From 1482601ae5c77cc1d54f6e4c7731dd44b3a1d738 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 09:26:24 +0000 Subject: [PATCH] =?UTF-8?q?Add=20num=5Fprompts=20=3D=205=C3=97concurrency?= =?UTF-8?q?=20convention=20and=20update=20all=20experiments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Document the rule in docs/EXPERIMENT_GUIDE.md section 7 - Update all config.env SCENARIOS to follow the 5x rule - Migrate dsv4_h200_sglang, dsv4_h200_vllm, dsv4_p800_sglang from global NUM_PROMPTS to per-scenario num_prompts - Keep long-context phase2 as exception (low counts documented) - Add legacy 3-field fallback in run_bench.sh loops --- docs/EXPERIMENT_GUIDE.md | 21 ++++++++- experiments/TEMPLATE/config.env | 5 ++- .../dsv4_h200_256k_4k_probe/config.env | 3 +- .../dsv4_h200_64k_sglang_vs_vllm/config.env | 5 ++- .../dsv4_h200_long_context_matrix/config.env | 44 ++++++++++--------- experiments/dsv4_h200_sglang/config.env | 10 +++-- experiments/dsv4_h200_sglang/run_bench.sh | 13 +++--- .../dsv4_h200_sglang_vs_vllm/config.env | 16 ++++--- experiments/dsv4_h200_vllm/config.env | 14 +++--- experiments/dsv4_h200_vllm/run_bench.sh | 13 +++--- .../config.env | 11 ++--- .../dsv4_h200_vllm_mtp_vs_default/config.env | 13 +++--- experiments/dsv4_p800_sglang/config.env | 10 ++--- experiments/dsv4_p800_sglang/run_bench.sh | 12 +++-- 14 files changed, 116 insertions(+), 74 deletions(-) diff --git a/docs/EXPERIMENT_GUIDE.md b/docs/EXPERIMENT_GUIDE.md index 9609f5e..a772634 100644 --- a/docs/EXPERIMENT_GUIDE.md +++ b/docs/EXPERIMENT_GUIDE.md @@ -155,14 +155,31 @@ git commit -m "results: run " git push origin main ``` -## 7. 命名约定 +## 7. 场景设计规范 + +### 7.1 请求数 = 并发数 × 5 + +每个 scenario 的 `num_prompts` 必须是对应 `concurrency` 的 **5 倍**，即每个并发档位至少跑满 5 轮请求，保证 P95/P99 等尾延迟指标有统计意义。 + +```bash +# 格式：concurrency input_len output_len num_prompts +declare -a SCENARIOS=( + "1 512 256 5" # 1 × 5 + "32 512 256 160" # 32 × 5 + "128 512 256 640" # 128 × 5 +) +``` + +例外：对于超长上下文（如 128k/256k）或探针类实验，若执行成本过高，可在 `config.env` 中显式注释说明原因，并保留不小于 `concurrency × 2` 的最低样本量。 + +## 8. 命名约定 - 实验目录：`{model}_{chip}_{scenario}_{backend_vs_backend}`，例如 `dsv4_h200_sglang_vs_vllm` - scenario 名称：`c{concurrency}_i{input_len}_o{output_len}` - raw 输出：`{backend}_{label}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl` - run_id：`YYYYMMDD-HHMMSS` -## 8. 常见问题 +## 9. 常见问题 ### 8.1 新芯片上 server 起不来 diff --git a/experiments/TEMPLATE/config.env b/experiments/TEMPLATE/config.env index 8d6df3c..f75bcad 100644 --- a/experiments/TEMPLATE/config.env +++ b/experiments/TEMPLATE/config.env @@ -22,9 +22,10 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" MAX_RUNNING="${MAX_RUNNING:-256}" # Scenarios: "concurrency input_len output_len num_prompts". +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. declare -a SCENARIOS=( - "1 512 256 32" - "32 512 256 128" + "1 512 256 5" + "32 512 256 160" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_256k_4k_probe/config.env b/experiments/dsv4_h200_256k_4k_probe/config.env index b9a72e6..2994862 100644 --- a/experiments/dsv4_h200_256k_4k_probe/config.env +++ b/experiments/dsv4_h200_256k_4k_probe/config.env @@ -22,8 +22,9 @@ MAX_NUM_SEQS=8 MAX_RUNNING=8 # "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. declare -a SCENARIOS=( - "2 262144 4096 4" + "2 262144 4096 10" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env b/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env index c64aff4..d4caad7 100644 --- a/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env +++ b/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env @@ -23,9 +23,10 @@ MAX_NUM_SEQS=64 MAX_RUNNING=64 # "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. declare -a SCENARIOS=( - "25 65536 256 100" - "8 65536 1024 100" + "25 65536 256 125" + "8 65536 1024 40" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_long_context_matrix/config.env b/experiments/dsv4_h200_long_context_matrix/config.env index fcddcf5..37f9325 100644 --- a/experiments/dsv4_h200_long_context_matrix/config.env +++ b/experiments/dsv4_h200_long_context_matrix/config.env @@ -32,34 +32,36 @@ declare -a CONTEXT_GROUPS=( ) # Scenarios per group: "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. +# NOTE: long-context scenarios are expensive; these counts follow the 5x rule. declare -a SCENARIOS_64K=( - "25 65536 256 50" - "25 65536 1024 50" - "25 65536 4096 50" + "25 65536 256 125" + "25 65536 1024 125" + "25 65536 4096 125" ) declare -a SCENARIOS_128K=( - "10 131072 256 20" - "5 131072 256 10" - "2 131072 256 4" - "10 131072 1024 20" - "5 131072 1024 10" - "2 131072 1024 4" - "10 131072 4096 20" - "5 131072 4096 10" - "2 131072 4096 4" + "10 131072 256 50" + "5 131072 256 25" + "2 131072 256 10" + "10 131072 1024 50" + "5 131072 1024 25" + "2 131072 1024 10" + "10 131072 4096 50" + "5 131072 4096 25" + "2 131072 4096 10" ) declare -a SCENARIOS_256K=( - "10 262144 256 20" - "5 262144 256 10" - "2 262144 256 4" - "10 262144 1024 20" - "5 262144 1024 10" - "2 262144 1024 4" - "10 262144 4096 20" - "5 262144 4096 10" - "2 262144 4096 4" + "10 262144 256 50" + "5 262144 256 25" + "2 262144 256 10" + "10 262144 1024 50" + "5 262144 1024 25" + "2 262144 1024 10" + "10 262144 4096 50" + "5 262144 4096 25" + "2 262144 4096 10" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_sglang/config.env b/experiments/dsv4_h200_sglang/config.env index 2fc51bb..60e0f43 100644 --- a/experiments/dsv4_h200_sglang/config.env +++ b/experiments/dsv4_h200_sglang/config.env @@ -15,14 +15,16 @@ ENGINE="sglang" VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/sglang}" VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# Benchmark scenarios: each element is "concurrency input_len output_len". +# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts". +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. if [[ -z "${SCENARIOS:-}" ]]; then SCENARIOS=( - "32 512 256" - "128 512 256" + "32 512 256 160" + "128 512 256 640" ) fi -NUM_PROMPTS="${NUM_PROMPTS:-128}" +# Legacy fallback: used only if SCENARIOS entries have 3 fields. +NUM_PROMPTS="${NUM_PROMPTS:-160}" # Server start script bundled with this experiment. SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_sglang/run_bench.sh b/experiments/dsv4_h200_sglang/run_bench.sh index 18b4044..8c494b1 100755 --- a/experiments/dsv4_h200_sglang/run_bench.sh +++ b/experiments/dsv4_h200_sglang/run_bench.sh @@ -60,8 +60,7 @@ data["config"] = { "tp": 8, "moe_runner_backend": "marlin", "port": 30006, - "num_prompts": 128, - "scenarios": ["32 512 256", "128 512 256"] + "scenarios": ["32 512 256 160", "128 512 256 640"] } with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) @@ -126,11 +125,15 @@ fi log "===== BENCHMARK START =====" for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len <<< "$scenario" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi output_file="${RAW_DIR}/sglang_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log" - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ --backend sglang \ @@ -139,7 +142,7 @@ for scenario in "${SCENARIOS[@]}"; do --dataset-name random \ --random-input-len "$input_len" \ --random-output-len "$output_len" \ - --num-prompts "$NUM_PROMPTS" \ + --num-prompts "$num_prompts" \ --max-concurrency "$concurrency" \ --request-rate 10000 \ --output-file "$output_file" \ diff --git a/experiments/dsv4_h200_sglang_vs_vllm/config.env b/experiments/dsv4_h200_sglang_vs_vllm/config.env index 7d1708f..d7c7c07 100644 --- a/experiments/dsv4_h200_sglang_vs_vllm/config.env +++ b/experiments/dsv4_h200_sglang_vs_vllm/config.env @@ -20,13 +20,14 @@ TP=8 # Phase 1: short-context throughput (max_model_len 32k). # Each element: "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. if [[ -z "${PHASE1_SCENARIOS:-}" ]]; then declare -a PHASE1_SCENARIOS=( - "1 512 256 32" - "32 512 256 128" - "128 512 256 128" - "1 4000 512 32" - "32 4000 512 64" + "1 512 256 5" + "32 512 256 160" + "128 512 256 640" + "1 4000 512 5" + "32 4000 512 160" ) fi PHASE1_MAX_MODEL_LEN=32768 @@ -34,6 +35,7 @@ PHASE1_MAX_NUM_SEQS=256 PHASE1_MAX_RUNNING=256 # Phase 2: long context (max_model_len 210k to leave room for output + padding). +# Exception to the 5x rule due to very long input lengths; kept at low counts. if [[ -z "${PHASE2_SCENARIOS:-}" ]]; then declare -a PHASE2_SCENARIOS=( "1 32768 1024 8" @@ -51,8 +53,8 @@ SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_ SHAREGPT_CONTEXT_LEN=32768 if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then declare -a SHAREGPT_SCENARIOS=( - "1 32768 256 32" - "32 32768 256 128" + "1 32768 256 5" + "32 32768 256 160" ) fi diff --git a/experiments/dsv4_h200_vllm/config.env b/experiments/dsv4_h200_vllm/config.env index cda9047..46f1073 100644 --- a/experiments/dsv4_h200_vllm/config.env +++ b/experiments/dsv4_h200_vllm/config.env @@ -15,17 +15,19 @@ ENGINE="vllm" VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}" VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# Benchmark scenarios: each element is "concurrency input_len output_len". +# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts". +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. # Override via SCENARIOS env var as a bash array, e.g.: -# SCENARIOS=("32 512 256" "128 512 256") +# SCENARIOS=("32 512 256 160" "128 512 256 640") if [[ -z "${SCENARIOS:-}" ]]; then SCENARIOS=( - "32 512 256" - "128 512 256" - "32 4000 512" + "32 512 256 160" + "128 512 256 640" + "32 4000 512 160" ) fi -NUM_PROMPTS="${NUM_PROMPTS:-128}" +# Legacy fallback: used only if SCENARIOS entries have 3 fields. +NUM_PROMPTS="${NUM_PROMPTS:-160}" # Server start script bundled with this experiment. SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm/run_bench.sh b/experiments/dsv4_h200_vllm/run_bench.sh index 12f5245..257f57b 100755 --- a/experiments/dsv4_h200_vllm/run_bench.sh +++ b/experiments/dsv4_h200_vllm/run_bench.sh @@ -62,8 +62,7 @@ data["config"] = { "block_size": 256, "max_num_seqs": 256, "port": 30005, - "num_prompts": 128, - "scenarios": ["32 512 256", "128 512 256", "32 4000 512"] + "scenarios": ["32 512 256 160", "128 512 256 640", "32 4000 512 160"] } with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) @@ -128,11 +127,15 @@ fi log "===== BENCHMARK START =====" for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len <<< "$scenario" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ --backend vllm \ @@ -141,7 +144,7 @@ for scenario in "${SCENARIOS[@]}"; do --dataset-name random \ --random-input-len "$input_len" \ --random-output-len "$output_len" \ - --num-prompts "$NUM_PROMPTS" \ + --num-prompts "$num_prompts" \ --max-concurrency "$concurrency" \ --request-rate 10000 \ --output-file "$output_file" \ diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/config.env b/experiments/dsv4_h200_vllm_dspark_vs_default/config.env index ce532b5..e900986 100644 --- a/experiments/dsv4_h200_vllm_dspark_vs_default/config.env +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/config.env @@ -30,13 +30,14 @@ TP=8 # Scenarios focused on TTFT: low/high concurrency x short/medium input. # Each element: "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. if [[ -z "${SCENARIOS:-}" ]]; then declare -a SCENARIOS=( - "1 512 256 32" - "32 512 256 128" - "128 512 256 128" - "1 4000 512 32" - "32 4000 512 64" + "1 512 256 5" + "32 512 256 160" + "128 512 256 640" + "1 4000 512 5" + "32 4000 512 160" ) fi MAX_MODEL_LEN=32768 diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/config.env b/experiments/dsv4_h200_vllm_mtp_vs_default/config.env index a6ee34f..34c73e8 100644 --- a/experiments/dsv4_h200_vllm_mtp_vs_default/config.env +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/config.env @@ -21,12 +21,15 @@ SPEC_METHOD="${SPEC_METHOD:-mtp}" SPEC_TOKENS="${SPEC_TOKENS:-1}" # Scenarios must match the ones in the reused default result set. +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. +# NOTE: if these scenarios are changed, the default-side results must be re-run +# with the same scenarios (update DEFAULT_RESULT_ROOT accordingly). declare -a SCENARIOS=( - "1 512 256 32" - "32 512 256 128" - "128 512 256 128" - "1 4000 512 32" - "32 4000 512 64" + "1 512 256 5" + "32 512 256 160" + "128 512 256 640" + "1 4000 512 5" + "32 4000 512 160" ) MTP_START_SCRIPT="${SCRIPT_DIR:-.}/start_mtp.sh" diff --git a/experiments/dsv4_p800_sglang/config.env b/experiments/dsv4_p800_sglang/config.env index f984eb8..919bcaa 100644 --- a/experiments/dsv4_p800_sglang/config.env +++ b/experiments/dsv4_p800_sglang/config.env @@ -24,13 +24,13 @@ NUM_PROMPTS="${NUM_PROMPTS:-10}" # - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint SERVER_MODE="${SERVER_MODE:-w8a8_int8}" -# Scenario list: "concurrency input_len output_len" (one per line). -# Default to a single smoke-test scenario for fast validation. Override to run -# a full grid, e.g.: -# SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256") +# Scenario list: "concurrency input_len output_len num_prompts" (one per line). +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. +# Default to a single scenario for fast validation. Override to run a full grid, e.g.: +# SCENARIOS=("32 512 256 160" "128 512 256 640") if [[ -z "${SCENARIOS:-}" ]]; then SCENARIOS=( - "32 512 256" + "32 512 256 160" ) fi diff --git a/experiments/dsv4_p800_sglang/run_bench.sh b/experiments/dsv4_p800_sglang/run_bench.sh index 3b8d29e..bbd3abe 100755 --- a/experiments/dsv4_p800_sglang/run_bench.sh +++ b/experiments/dsv4_p800_sglang/run_bench.sh @@ -72,15 +72,19 @@ trap on_exit EXIT log "===== BENCHMARK START =====" for scenario in "${SCENARIOS[@]}"; do - # Allow SCENARIOS to be passed as a string like "(32 512 256)" from env. + # Allow SCENARIOS to be passed as a string like "(32 512 256 160)" from env. scenario="${scenario//[()]/}" - read -r concurrency input_len output_len <<< "$scenario" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" host_output_file="${RAW_DIR}/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/${BACKEND}_c${concurrency}_i${input_len}_o${output_len}.log" - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" run_random_case \ "$BACKEND" \ @@ -90,7 +94,7 @@ for scenario in "${SCENARIOS[@]}"; do "$concurrency" \ "$input_len" \ "$output_len" \ - "$NUM_PROMPTS" \ + "$num_prompts" \ "${DATASET_PATH:-}" \ > "$detail_log" 2>&1 || { log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed"