Add num_prompts = 5×concurrency convention and update all experiments

- Document the rule in docs/EXPERIMENT_GUIDE.md section 7
- Update all config.env SCENARIOS to follow the 5x rule
- Migrate dsv4_h200_sglang, dsv4_h200_vllm, dsv4_p800_sglang
  from global NUM_PROMPTS to per-scenario num_prompts
- Keep long-context phase2 as exception (low counts documented)
- Add legacy 3-field fallback in run_bench.sh loops

docs/EXPERIMENT_GUIDE.md

@@ -155,14 +155,31 @@ git commit -m "results: <experiment_name> run <run_id>"
 git push origin main
 ```
 
-## 7. 命名约定
+## 7. 场景设计规范
+
+### 7.1 请求数 = 并发数 × 5
+
+每个 scenario 的 `num_prompts` 必须是对应 `concurrency` 的 **5 倍**，即每个并发档位至少跑满 5 轮请求，保证 P95/P99 等尾延迟指标有统计意义。
+
+```bash
+# 格式：concurrency input_len output_len num_prompts
+declare -a SCENARIOS=(
+  "1   512  256  5"    # 1 × 5
+  "32  512  256  160"  # 32 × 5
+  "128 512  256  640"  # 128 × 5
+)
+```
+
+例外：对于超长上下文（如 128k/256k）或探针类实验，若执行成本过高，可在 `config.env` 中显式注释说明原因，并保留不小于 `concurrency × 2` 的最低样本量。
+
+## 8. 命名约定
 
 - 实验目录：`{model}_{chip}_{scenario}_{backend_vs_backend}`，例如 `dsv4_h200_sglang_vs_vllm`
 - scenario 名称：`c{concurrency}_i{input_len}_o{output_len}`
 - raw 输出：`{backend}_{label}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl`
 - run_id：`YYYYMMDD-HHMMSS`
 
-## 8. 常见问题
+## 9. 常见问题
 
 ### 8.1 新芯片上 server 起不来
 

experiments/TEMPLATE/config.env

@@ -22,9 +22,10 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
 MAX_RUNNING="${MAX_RUNNING:-256}"
 
 # Scenarios: "concurrency input_len output_len num_prompts".
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 declare -a SCENARIOS=(
-  "1  512  256 32"
+  "1  512  256 5"
-  "32 512  256 128"
+  "32 512  256 160"
 )
 
 VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

experiments/dsv4_h200_256k_4k_probe/config.env

@@ -22,8 +22,9 @@ MAX_NUM_SEQS=8
 MAX_RUNNING=8
 
 # "concurrency input_len output_len num_prompts"
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 declare -a SCENARIOS=(
-  "2 262144 4096 4"
+  "2 262144 4096 10"
 )
 
 VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

experiments/dsv4_h200_64k_sglang_vs_vllm/config.env

@@ -23,9 +23,10 @@ MAX_NUM_SEQS=64
 MAX_RUNNING=64
 
 # "concurrency input_len output_len num_prompts"
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 declare -a SCENARIOS=(
-  "25 65536 256  100"
+  "25 65536 256  125"
-  "8  65536 1024 100"
+  "8  65536 1024 40"
 )
 
 VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

experiments/dsv4_h200_long_context_matrix/config.env

@@ -32,34 +32,36 @@ declare -a CONTEXT_GROUPS=(
 )
 
 # Scenarios per group: "concurrency input_len output_len num_prompts"
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
+# NOTE: long-context scenarios are expensive; these counts follow the 5x rule.
 declare -a SCENARIOS_64K=(
-  "25 65536  256  50"
+  "25 65536  256  125"
-  "25 65536  1024 50"
+  "25 65536  1024 125"
-  "25 65536  4096 50"
+  "25 65536  4096 125"
 )
 
 declare -a SCENARIOS_128K=(
-  "10 131072 256  20"
+  "10 131072 256  50"
-  "5  131072 256  10"
+  "5  131072 256  25"
-  "2  131072 256  4"
+  "2  131072 256  10"
-  "10 131072 1024 20"
+  "10 131072 1024 50"
-  "5  131072 1024 10"
+  "5  131072 1024 25"
-  "2  131072 1024 4"
+  "2  131072 1024 10"
-  "10 131072 4096 20"
+  "10 131072 4096 50"
-  "5  131072 4096 10"
+  "5  131072 4096 25"
-  "2  131072 4096 4"
+  "2  131072 4096 10"
 )
 
 declare -a SCENARIOS_256K=(
-  "10 262144 256  20"
+  "10 262144 256  50"
-  "5  262144 256  10"
+  "5  262144 256  25"
-  "2  262144 256  4"
+  "2  262144 256  10"
-  "10 262144 1024 20"
+  "10 262144 1024 50"
-  "5  262144 1024 10"
+  "5  262144 1024 25"
-  "2  262144 1024 4"
+  "2  262144 1024 10"
-  "10 262144 4096 20"
+  "10 262144 4096 50"
-  "5  262144 4096 10"
+  "5  262144 4096 25"
-  "2  262144 4096 4"
+  "2  262144 4096 10"
 )
 
 VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

experiments/dsv4_h200_sglang/config.env

@@ -15,14 +15,16 @@ ENGINE="sglang"
 VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/sglang}"
 VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
 
-# Benchmark scenarios: each element is "concurrency input_len output_len".
+# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts".
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 if [[ -z "${SCENARIOS:-}" ]]; then
   SCENARIOS=(
-    "32 512 256"
+    "32 512 256 160"
-    "128 512 256"
+    "128 512 256 640"
   )
 fi
-NUM_PROMPTS="${NUM_PROMPTS:-128}"
+# Legacy fallback: used only if SCENARIOS entries have 3 fields.
+NUM_PROMPTS="${NUM_PROMPTS:-160}"
 
 # Server start script bundled with this experiment.
 SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

experiments/dsv4_h200_sglang/run_bench.sh

@@ -60,8 +60,7 @@ data["config"] = {
     "tp": 8,
     "moe_runner_backend": "marlin",
     "port": 30006,
-    "num_prompts": 128,
+    "scenarios": ["32 512 256 160", "128 512 256 640"]
-    "scenarios": ["32 512 256", "128 512 256"]
 }
 with open(path, "w", encoding="utf-8") as f:
     json.dump(data, f, indent=2, ensure_ascii=False)
@@ -126,11 +125,15 @@ fi
 log "===== BENCHMARK START ====="
 
 for scenario in "${SCENARIOS[@]}"; do
-  read -r concurrency input_len output_len <<< "$scenario"
+  read -r concurrency input_len output_len num_prompts <<< "$scenario"
+  # Fallback for legacy 3-field scenarios.
+  if [[ -z "${num_prompts:-}" ]]; then
+    num_prompts="$NUM_PROMPTS"
+  fi
   output_file="${RAW_DIR}/sglang_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
   detail_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log"
 
-  log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}"
+  log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
 
   "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
     --backend sglang \
@@ -139,7 +142,7 @@ for scenario in "${SCENARIOS[@]}"; do
     --dataset-name random \
     --random-input-len "$input_len" \
     --random-output-len "$output_len" \
-    --num-prompts "$NUM_PROMPTS" \
+    --num-prompts "$num_prompts" \
     --max-concurrency "$concurrency" \
     --request-rate 10000 \
     --output-file "$output_file" \

experiments/dsv4_h200_sglang_vs_vllm/config.env

@@ -20,13 +20,14 @@ TP=8
 
 # Phase 1: short-context throughput (max_model_len 32k).
 # Each element: "concurrency input_len output_len num_prompts"
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 if [[ -z "${PHASE1_SCENARIOS:-}" ]]; then
   declare -a PHASE1_SCENARIOS=(
-    "1   512   256  32"
+    "1   512   256  5"
-    "32  512   256  128"
+    "32  512   256  160"
-    "128 512   256  128"
+    "128 512   256  640"
-    "1   4000  512  32"
+    "1   4000  512  5"
-    "32  4000  512  64"
+    "32  4000  512  160"
   )
 fi
 PHASE1_MAX_MODEL_LEN=32768
@@ -34,6 +35,7 @@ PHASE1_MAX_NUM_SEQS=256
 PHASE1_MAX_RUNNING=256
 
 # Phase 2: long context (max_model_len 210k to leave room for output + padding).
+# Exception to the 5x rule due to very long input lengths; kept at low counts.
 if [[ -z "${PHASE2_SCENARIOS:-}" ]]; then
   declare -a PHASE2_SCENARIOS=(
     "1  32768   1024  8"
@@ -51,8 +53,8 @@ SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_
 SHAREGPT_CONTEXT_LEN=32768
 if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then
   declare -a SHAREGPT_SCENARIOS=(
-    "1  32768  256  32"
+    "1  32768  256  5"
-    "32 32768  256  128"
+    "32 32768  256  160"
   )
 fi
 

experiments/dsv4_h200_vllm/config.env

@@ -15,17 +15,19 @@ ENGINE="vllm"
 VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}"
 VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
 
-# Benchmark scenarios: each element is "concurrency input_len output_len".
+# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts".
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 # Override via SCENARIOS env var as a bash array, e.g.:
-#   SCENARIOS=("32 512 256" "128 512 256")
+#   SCENARIOS=("32 512 256 160" "128 512 256 640")
 if [[ -z "${SCENARIOS:-}" ]]; then
   SCENARIOS=(
-    "32 512 256"
+    "32 512 256 160"
-    "128 512 256"
+    "128 512 256 640"
-    "32 4000 512"
+    "32 4000 512 160"
   )
 fi
-NUM_PROMPTS="${NUM_PROMPTS:-128}"
+# Legacy fallback: used only if SCENARIOS entries have 3 fields.
+NUM_PROMPTS="${NUM_PROMPTS:-160}"
 
 # Server start script bundled with this experiment.
 SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

experiments/dsv4_h200_vllm/run_bench.sh

@@ -62,8 +62,7 @@ data["config"] = {
     "block_size": 256,
     "max_num_seqs": 256,
     "port": 30005,
-    "num_prompts": 128,
+    "scenarios": ["32 512 256 160", "128 512 256 640", "32 4000 512 160"]
-    "scenarios": ["32 512 256", "128 512 256", "32 4000 512"]
 }
 with open(path, "w", encoding="utf-8") as f:
     json.dump(data, f, indent=2, ensure_ascii=False)
@@ -128,11 +127,15 @@ fi
 log "===== BENCHMARK START ====="
 
 for scenario in "${SCENARIOS[@]}"; do
-  read -r concurrency input_len output_len <<< "$scenario"
+  read -r concurrency input_len output_len num_prompts <<< "$scenario"
+  # Fallback for legacy 3-field scenarios.
+  if [[ -z "${num_prompts:-}" ]]; then
+    num_prompts="$NUM_PROMPTS"
+  fi
   output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
   detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
 
-  log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}"
+  log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
 
   "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
     --backend vllm \
@@ -141,7 +144,7 @@ for scenario in "${SCENARIOS[@]}"; do
     --dataset-name random \
     --random-input-len "$input_len" \
     --random-output-len "$output_len" \
-    --num-prompts "$NUM_PROMPTS" \
+    --num-prompts "$num_prompts" \
     --max-concurrency "$concurrency" \
     --request-rate 10000 \
     --output-file "$output_file" \

experiments/dsv4_h200_vllm_dspark_vs_default/config.env

@@ -30,13 +30,14 @@ TP=8
 
 # Scenarios focused on TTFT: low/high concurrency x short/medium input.
 # Each element: "concurrency input_len output_len num_prompts"
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
 if [[ -z "${SCENARIOS:-}" ]]; then
   declare -a SCENARIOS=(
-    "1   512   256  32"
+    "1   512   256  5"
-    "32  512   256  128"
+    "32  512   256  160"
-    "128 512   256  128"
+    "128 512   256  640"
-    "1   4000  512  32"
+    "1   4000  512  5"
-    "32  4000  512  64"
+    "32  4000  512  160"
   )
 fi
 MAX_MODEL_LEN=32768

experiments/dsv4_h200_vllm_mtp_vs_default/config.env

@@ -21,12 +21,15 @@ SPEC_METHOD="${SPEC_METHOD:-mtp}"
 SPEC_TOKENS="${SPEC_TOKENS:-1}"
 
 # Scenarios must match the ones in the reused default result set.
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
+# NOTE: if these scenarios are changed, the default-side results must be re-run
+# with the same scenarios (update DEFAULT_RESULT_ROOT accordingly).
 declare -a SCENARIOS=(
-  "1   512   256  32"
+  "1   512   256  5"
-  "32  512   256  128"
+  "32  512   256  160"
-  "128 512   256  128"
+  "128 512   256  640"
-  "1   4000  512  32"
+  "1   4000  512  5"
-  "32  4000  512  64"
+  "32  4000  512  160"
 )
 
 MTP_START_SCRIPT="${SCRIPT_DIR:-.}/start_mtp.sh"

experiments/dsv4_p800_sglang/config.env

@@ -24,13 +24,13 @@ NUM_PROMPTS="${NUM_PROMPTS:-10}"
 #   - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint
 SERVER_MODE="${SERVER_MODE:-w8a8_int8}"
 
-# Scenario list: "concurrency input_len output_len" (one per line).
+# Scenario list: "concurrency input_len output_len num_prompts" (one per line).
-# Default to a single smoke-test scenario for fast validation. Override to run
+# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5.
-# a full grid, e.g.:
+# Default to a single scenario for fast validation. Override to run a full grid, e.g.:
-#   SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256")
+#   SCENARIOS=("32 512 256 160" "128 512 256 640")
 if [[ -z "${SCENARIOS:-}" ]]; then
   SCENARIOS=(
-    "32 512 256"
+    "32 512 256 160"
   )
 fi
 

experiments/dsv4_p800_sglang/run_bench.sh

@@ -72,15 +72,19 @@ trap on_exit EXIT
 log "===== BENCHMARK START ====="
 
 for scenario in "${SCENARIOS[@]}"; do
-  # Allow SCENARIOS to be passed as a string like "(32 512 256)" from env.
+  # Allow SCENARIOS to be passed as a string like "(32 512 256 160)" from env.
   scenario="${scenario//[()]/}"
-  read -r concurrency input_len output_len <<< "$scenario"
+  read -r concurrency input_len output_len num_prompts <<< "$scenario"
+  # Fallback for legacy 3-field scenarios.
+  if [[ -z "${num_prompts:-}" ]]; then
+    num_prompts="$NUM_PROMPTS"
+  fi
 
   tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
   host_output_file="${RAW_DIR}/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
   detail_log="${LOG_DIR}/${BACKEND}_c${concurrency}_i${input_len}_o${output_len}.log"
 
-  log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}"
+  log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
 
   run_random_case \
     "$BACKEND" \
@@ -90,7 +94,7 @@ for scenario in "${SCENARIOS[@]}"; do
     "$concurrency" \
     "$input_len" \
     "$output_len" \
-    "$NUM_PROMPTS" \
+    "$num_prompts" \
     "${DATASET_PATH:-}" \
     > "$detail_log" 2>&1 || {
       log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed"