qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add num_prompts = 5×concurrency convention and update all experiments

Browse Source 
- Document the rule in docs/EXPERIMENT_GUIDE.md section 7
- Update all config.env SCENARIOS to follow the 5x rule
- Migrate dsv4_h200_sglang, dsv4_h200_vllm, dsv4_p800_sglang
  from global NUM_PROMPTS to per-scenario num_prompts
- Keep long-context phase2 as exception (low counts documented)
- Add legacy 3-field fallback in run_bench.sh loops
This commit is contained in:
yy-fighting 2026-07-09 09:26:24 +00:00
parent a1da95ef44
commit 1482601ae5
14 changed files with 116 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
docs/EXPERIMENT_GUIDE.md
View File

@ -155,14 +155,31 @@ git commit -m "results: <experiment_name> run <run_id>"
git push origin main
```
## 7. 命名约定
## 7. 场景设计规范
### 7.1 请求数 = 并发数 × 5
每个 scenario 的 `num_prompts` 必须是对应 `concurrency` 的 **5 倍**，即每个并发档位至少跑满 5 轮请求，保证 P95/P99 等尾延迟指标有统计意义。
```bash
# 格式：concurrency input_len output_len num_prompts
declare -a SCENARIOS=(
"1 512 256 5" # 1 × 5
"32 512 256 160" # 32 × 5
"128 512 256 640" # 128 × 5
)
```
例外：对于超长上下文（如 128k/256k）或探针类实验，若执行成本过高，可在 `config.env` 中显式注释说明原因，并保留不小于 `concurrency × 2` 的最低样本量。
## 8. 命名约定
- 实验目录：`{model}_{chip}_{scenario}_{backend_vs_backend}`，例如 `dsv4_h200_sglang_vs_vllm`
- scenario 名称：`c{concurrency}_i{input_len}_o{output_len}`
- raw 输出：`{backend}_{label}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl`
- run_id：`YYYYMMDD-HHMMSS`
## 8. 常见问题
## 9. 常见问题
### 8.1 新芯片上 server 起不来

5
experiments/TEMPLATE/config.env
View File

@ -22,9 +22,10 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
MAX_RUNNING="${MAX_RUNNING:-256}"
# Scenarios: "concurrency input_len output_len num_prompts".
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
declare -a SCENARIOS=(
"1 512 256 32"
"32 512 256 128"
"1 512 256 5"
"32 512 256 160"
)
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

3
experiments/dsv4_h200_256k_4k_probe/config.env
View File

@ -22,8 +22,9 @@ MAX_NUM_SEQS=8
MAX_RUNNING=8
# "concurrency input_len output_len num_prompts"
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
declare -a SCENARIOS=(
"2 262144 4096 4"
"2 262144 4096 10"
)
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

5
experiments/dsv4_h200_64k_sglang_vs_vllm/config.env
View File

@ -23,9 +23,10 @@ MAX_NUM_SEQS=64
MAX_RUNNING=64
# "concurrency input_len output_len num_prompts"
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
declare -a SCENARIOS=(
"25 65536 256 100"
"8 65536 1024 100"
"25 65536 256 125"
"8 65536 1024 40"
)
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

44
experiments/dsv4_h200_long_context_matrix/config.env
View File

@ -32,34 +32,36 @@ declare -a CONTEXT_GROUPS=(
)
# Scenarios per group: "concurrency input_len output_len num_prompts"
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
# NOTE: long-context scenarios are expensive; these counts follow the 5x rule.
declare -a SCENARIOS_64K=(
"25 65536 256 50"
"25 65536 1024 50"
"25 65536 4096 50"
"25 65536 256 125"
"25 65536 1024 125"
"25 65536 4096 125"
)
declare -a SCENARIOS_128K=(
"10 131072 256 20"
"5 131072 256 10"
"2 131072 256 4"
"10 131072 1024 20"
"5 131072 1024 10"
"2 131072 1024 4"
"10 131072 4096 20"
"5 131072 4096 10"
"2 131072 4096 4"
"10 131072 256 50"
"5 131072 256 25"
"2 131072 256 10"
"10 131072 1024 50"
"5 131072 1024 25"
"2 131072 1024 10"
"10 131072 4096 50"
"5 131072 4096 25"
"2 131072 4096 10"
)
declare -a SCENARIOS_256K=(
"10 262144 256 20"
"5 262144 256 10"
"2 262144 256 4"
"10 262144 1024 20"
"5 262144 1024 10"
"2 262144 1024 4"
"10 262144 4096 20"
"5 262144 4096 10"
"2 262144 4096 4"
"10 262144 256 50"
"5 262144 256 25"
"2 262144 256 10"
"10 262144 1024 50"
"5 262144 1024 25"
"2 262144 1024 10"
"10 262144 4096 50"
"5 262144 4096 25"
"2 262144 4096 10"
)
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

10
experiments/dsv4_h200_sglang/config.env
View File

@ -15,14 +15,16 @@ ENGINE="sglang"
VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/sglang}"
VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
# Benchmark scenarios: each element is "concurrency input_len output_len".
# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts".
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
if [[ -z "${SCENARIOS:-}" ]]; then
SCENARIOS=(
"32 512 256"
"128 512 256"
"32 512 256 160"
"128 512 256 640"
)
fi
NUM_PROMPTS="${NUM_PROMPTS:-128}"
# Legacy fallback: used only if SCENARIOS entries have 3 fields.
NUM_PROMPTS="${NUM_PROMPTS:-160}"
# Server start script bundled with this experiment.
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

13
experiments/dsv4_h200_sglang/run_bench.sh
View File

@ -60,8 +60,7 @@ data["config"] = {
"tp": 8,
"moe_runner_backend": "marlin",
"port": 30006,
"num_prompts": 128,
"scenarios": ["32 512 256", "128 512 256"]
"scenarios": ["32 512 256 160", "128 512 256 640"]
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
@ -126,11 +125,15 @@ fi
log "===== BENCHMARK START ====="
for scenario in "${SCENARIOS[@]}"; do
read -r concurrency input_len output_len <<< "$scenario"
read -r concurrency input_len output_len num_prompts <<< "$scenario"
# Fallback for legacy 3-field scenarios.
if [[ -z "${num_prompts:-}" ]]; then
num_prompts="$NUM_PROMPTS"
fi
output_file="${RAW_DIR}/sglang_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
--backend sglang \
@ -139,7 +142,7 @@ for scenario in "${SCENARIOS[@]}"; do
--dataset-name random \
--random-input-len "$input_len" \
--random-output-len "$output_len" \
--num-prompts "$NUM_PROMPTS" \
--num-prompts "$num_prompts" \
--max-concurrency "$concurrency" \
--request-rate 10000 \
--output-file "$output_file" \

16
experiments/dsv4_h200_sglang_vs_vllm/config.env
View File

@ -20,13 +20,14 @@ TP=8
# Phase 1: short-context throughput (max_model_len 32k).
# Each element: "concurrency input_len output_len num_prompts"
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
if [[ -z "${PHASE1_SCENARIOS:-}" ]]; then
declare -a PHASE1_SCENARIOS=(
"1 512 256 32"
"32 512 256 128"
"128 512 256 128"
"1 4000 512 32"
"32 4000 512 64"
"1 512 256 5"
"32 512 256 160"
"128 512 256 640"
"1 4000 512 5"
"32 4000 512 160"
)
fi
PHASE1_MAX_MODEL_LEN=32768
@ -34,6 +35,7 @@ PHASE1_MAX_NUM_SEQS=256
PHASE1_MAX_RUNNING=256
# Phase 2: long context (max_model_len 210k to leave room for output + padding).
# Exception to the 5x rule due to very long input lengths; kept at low counts.
if [[ -z "${PHASE2_SCENARIOS:-}" ]]; then
declare -a PHASE2_SCENARIOS=(
"1 32768 1024 8"
@ -51,8 +53,8 @@ SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_
SHAREGPT_CONTEXT_LEN=32768
if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then
declare -a SHAREGPT_SCENARIOS=(
"1 32768 256 32"
"32 32768 256 128"
"1 32768 256 5"
"32 32768 256 160"
)
fi

14
experiments/dsv4_h200_vllm/config.env
View File

@ -15,17 +15,19 @@ ENGINE="vllm"
VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}"
VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
# Benchmark scenarios: each element is "concurrency input_len output_len".
# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts".
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
# Override via SCENARIOS env var as a bash array, e.g.:
# SCENARIOS=("32 512 256" "128 512 256")
# SCENARIOS=("32 512 256 160" "128 512 256 640")
if [[ -z "${SCENARIOS:-}" ]]; then
SCENARIOS=(
"32 512 256"
"128 512 256"
"32 4000 512"
"32 512 256 160"
"128 512 256 640"
"32 4000 512 160"
)
fi
NUM_PROMPTS="${NUM_PROMPTS:-128}"
# Legacy fallback: used only if SCENARIOS entries have 3 fields.
NUM_PROMPTS="${NUM_PROMPTS:-160}"
# Server start script bundled with this experiment.
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

13
experiments/dsv4_h200_vllm/run_bench.sh
View File

@ -62,8 +62,7 @@ data["config"] = {
"block_size": 256,
"max_num_seqs": 256,
"port": 30005,
"num_prompts": 128,
"scenarios": ["32 512 256", "128 512 256", "32 4000 512"]
"scenarios": ["32 512 256 160", "128 512 256 640", "32 4000 512 160"]
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
@ -128,11 +127,15 @@ fi
log "===== BENCHMARK START ====="
for scenario in "${SCENARIOS[@]}"; do
read -r concurrency input_len output_len <<< "$scenario"
read -r concurrency input_len output_len num_prompts <<< "$scenario"
# Fallback for legacy 3-field scenarios.
if [[ -z "${num_prompts:-}" ]]; then
num_prompts="$NUM_PROMPTS"
fi
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
--backend vllm \
@ -141,7 +144,7 @@ for scenario in "${SCENARIOS[@]}"; do
--dataset-name random \
--random-input-len "$input_len" \
--random-output-len "$output_len" \
--num-prompts "$NUM_PROMPTS" \
--num-prompts "$num_prompts" \
--max-concurrency "$concurrency" \
--request-rate 10000 \
--output-file "$output_file" \

11
experiments/dsv4_h200_vllm_dspark_vs_default/config.env
View File

@ -30,13 +30,14 @@ TP=8
# Scenarios focused on TTFT: low/high concurrency x short/medium input.
# Each element: "concurrency input_len output_len num_prompts"
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
if [[ -z "${SCENARIOS:-}" ]]; then
declare -a SCENARIOS=(
"1 512 256 32"
"32 512 256 128"
"128 512 256 128"
"1 4000 512 32"
"32 4000 512 64"
"1 512 256 5"
"32 512 256 160"
"128 512 256 640"
"1 4000 512 5"
"32 4000 512 160"
)
fi
MAX_MODEL_LEN=32768

13
experiments/dsv4_h200_vllm_mtp_vs_default/config.env
View File

@ -21,12 +21,15 @@ SPEC_METHOD="${SPEC_METHOD:-mtp}"
SPEC_TOKENS="${SPEC_TOKENS:-1}"
# Scenarios must match the ones in the reused default result set.
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
# NOTE: if these scenarios are changed, the default-side results must be re-run
# with the same scenarios (update DEFAULT_RESULT_ROOT accordingly).
declare -a SCENARIOS=(
"1 512 256 32"
"32 512 256 128"
"128 512 256 128"
"1 4000 512 32"
"32 4000 512 64"
"1 512 256 5"
"32 512 256 160"
"128 512 256 640"
"1 4000 512 5"
"32 4000 512 160"
)
MTP_START_SCRIPT="${SCRIPT_DIR:-.}/start_mtp.sh"

10
experiments/dsv4_p800_sglang/config.env
View File

@ -24,13 +24,13 @@ NUM_PROMPTS="${NUM_PROMPTS:-10}"
# - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint
SERVER_MODE="${SERVER_MODE:-w8a8_int8}"
# Scenario list: "concurrency input_len output_len" (one per line).
# Default to a single smoke-test scenario for fast validation. Override to run
# a full grid, e.g.:
# SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256")
# Scenario list: "concurrency input_len output_len num_prompts" (one per line).
# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
# Default to a single scenario for fast validation. Override to run a full grid, e.g.:
# SCENARIOS=("32 512 256 160" "128 512 256 640")
if [[ -z "${SCENARIOS:-}" ]]; then
SCENARIOS=(
"32 512 256"
"32 512 256 160"
)
fi

12
experiments/dsv4_p800_sglang/run_bench.sh
View File

@ -72,15 +72,19 @@ trap on_exit EXIT
log "===== BENCHMARK START ====="
for scenario in "${SCENARIOS[@]}"; do
# Allow SCENARIOS to be passed as a string like "(32 512 256)" from env.
# Allow SCENARIOS to be passed as a string like "(32 512 256 160)" from env.
scenario="${scenario//[()]/}"
read -r concurrency input_len output_len <<< "$scenario"
read -r concurrency input_len output_len num_prompts <<< "$scenario"
# Fallback for legacy 3-field scenarios.
if [[ -z "${num_prompts:-}" ]]; then
num_prompts="$NUM_PROMPTS"
fi
tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
host_output_file="${RAW_DIR}/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/${BACKEND}_c${concurrency}_i${input_len}_o${output_len}.log"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
run_random_case \
"$BACKEND" \
@ -90,7 +94,7 @@ for scenario in "${SCENARIOS[@]}"; do
"$concurrency" \
"$input_len" \
"$output_len" \
"$NUM_PROMPTS" \
"$num_prompts" \
"${DATASET_PATH:-}" \
> "$detail_log" 2>&1 || {
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed"