feat: add ShareGPT dataset scenarios to SGLang vs vLLM comparison

- config.env: add SHAREGPT_DATASET, SHAREGPT_CONTEXT_LEN and SHAREGPT_SCENARIOS
- run_bench.sh: add run_sharegpt() and integrate it after Phase 1 for each backend
- parse_backend.py: support both phase{1|2} and sharegpt raw output filenames
- README.md: document ShareGPT scenarios

experiments/dsv4_h200_sglang_vs_vllm/README.md

@@ -31,6 +31,15 @@
 | 1 | 131072 | 1024 | 2 |
 | 1 | 200000 | 1024 | 1 |
 
+### ShareGPT 真实对话数据（max_model_len=32k）
+
+| Concurrency | ShareGPT context len | Output | Num prompts |
+|---:|---:|---:|---:|
+| 1  | 32768 | 256 | 32  |
+| 32 | 32768 | 256 | 128 |
+
+使用 `datasets/ShareGPT_filtered_chat.json`，`--sharegpt-context-len 32768` 会自动过滤超长请求。
+
 ## 快速运行
 
 ```bash

experiments/dsv4_h200_sglang_vs_vllm/config.env

@@ -46,6 +46,16 @@ PHASE2_MAX_MODEL_LEN=210000
 PHASE2_MAX_NUM_SEQS=2
 PHASE2_MAX_RUNNING=2
 
+# ShareGPT dataset test (uses the same max_model_len as Phase 1).
+SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_chat.json}"
+SHAREGPT_CONTEXT_LEN=32768
+if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then
+  declare -a SHAREGPT_SCENARIOS=(
+    "1  32768  256  32"
+    "32 32768  256  128"
+  )
+fi
+
 # Benchmark client always runs from the sglang env so that the same
 # sglang.bench_serving version is used for both backends.
 VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"

experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py

@@ -140,16 +140,27 @@ def main() -> None:
     if backend is None:
         raise SystemExit("Could not infer backend from raw outputs")
 
-    # Filename: {backend}_phase{1|2}_MMDD_concurrency_inputlen_outputlen.jsonl
-    pattern = re.compile(rf"^{backend}_phase(\d+)_(\d+)_(\d+)_(\d+)_(\d+)\.jsonl$")
-
     scenarios = []
-    for jsonl_path in sorted(raw_dir.glob(f"{backend}_phase*.jsonl")):
+    for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
-        m = pattern.match(jsonl_path.name)
+        parts = jsonl_path.stem.split("_")
-        if not m:
+        if len(parts) < 5:
+            continue
+
+        label = parts[1]
+        if label.startswith("phase"):
+            phase = label
+            dataset = "random"
+        elif label == "sharegpt":
+            phase = "sharegpt"
+            dataset = "sharegpt"
+        else:
+            continue
+
+        # The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
+        try:
+            concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
+        except ValueError:
             continue
-        phase_num, concurrency, input_len, output_len = m.groups()
-        concurrency, input_len, output_len = int(concurrency), int(input_len), int(output_len)
 
         data = parse_jsonl(jsonl_path)
         if data is None:
@@ -159,11 +170,11 @@ def main() -> None:
         scenario = {
             "name": scenario_name(concurrency, input_len, output_len),
             "config": {
-                "phase": f"phase{phase_num}",
+                "phase": phase,
                 "concurrency": concurrency,
                 "input_len": input_len,
                 "output_len": output_len,
-                "dataset": "random",
+                "dataset": dataset,
                 "num_prompts": metrics["success"] + metrics["failed"],
             },
             "metrics": metrics,

experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh

@@ -174,6 +174,60 @@ run_phase() {
   log "===== ${backend} ${phase} DONE ====="
 }
 
+run_sharegpt() {
+  local backend="$1"
+  local phase="sharegpt"
+  local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
+  local raw_dir="${result_root}/raw_outputs"
+  local phase_log_dir="${result_root}/logs"
+
+  mkdir -p "$raw_dir" "$phase_log_dir"
+
+  local port
+  if [[ "$backend" == "sglang" ]]; then
+    port="$SGLANG_PORT"
+  else
+    port="$VLLM_PORT"
+  fi
+
+  log "===== ${backend} ${phase} START (context_len=${SHAREGPT_CONTEXT_LEN}) ====="
+
+  stop_server "$backend"
+  start_server "$backend" phase1
+  run_warmup "$backend" phase1
+
+  for scenario in "${SHAREGPT_SCENARIOS[@]}"; do
+    read -r concurrency context_len output_len num_prompts <<< "$scenario"
+    output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl"
+    detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log"
+
+    log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}"
+
+    "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
+      --backend "$backend" \
+      --host 127.0.0.1 \
+      --port "$port" \
+      --dataset-name sharegpt \
+      --dataset-path "$SHAREGPT_DATASET" \
+      --sharegpt-output-len "$output_len" \
+      --sharegpt-context-len "$context_len" \
+      --num-prompts "$num_prompts" \
+      --max-concurrency "$concurrency" \
+      --request-rate 10000 \
+      --output-file "$output_file" \
+      --output-details \
+      > "$detail_log" 2>&1 || {
+        log "ERROR: ${backend} ${phase} scenario c=${concurrency} ctx=${context_len} o=${output_len} failed; see ${detail_log}"
+        continue
+      }
+
+    log "finished ${backend} ${phase} scenario: output=${output_file}"
+  done
+
+  stop_server "$backend"
+  log "===== ${backend} ${phase} DONE ====="
+}
+
 parse_backend() {
   local backend="$1"
   local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
@@ -245,11 +299,13 @@ write_backend_metadata vllm
 
 # Run SGLang.
 run_phase sglang phase1
+run_sharegpt sglang
 run_phase sglang phase2
 parse_backend sglang
 
 # Run vLLM.
 run_phase vllm phase1
+run_sharegpt vllm
 run_phase vllm phase2
 parse_backend vllm