fix(dsv4_h200_vllm): scenario array parsing and sglang output format

- Use bash array for SCENARIOS to avoid word-splitting
- Fix metadata config (tp=4, scenarios as array)
- Rewrite parse_results.py for sglang.bench_serving --output-details format
- Update .gitignore to keep experiments/*/results/*.json and *.md,
  ignore only raw_outputs/ and logs/ subdirs
- First successful run: 20260708-062348

.gitignore

@@ -23,8 +23,9 @@ datasets/
 # 原始请求级输出（方案 A：极简版，不存原始 jsonl）
 bench_results/**/raw_outputs/
 
-# 实验级原始结果目录（report.md / results.json 可单独保留，默认忽略整个目录）
+# 实验级原始结果目录：保留 report.md / results.json，忽略原始 jsonl 和日志
-experiments/*/results/
+experiments/*/results/raw_outputs/
+experiments/*/results/logs/
 
 # 无关项目
 loomeval_yy/

experiments/dsv4_h200_vllm/config.env

@@ -15,9 +15,16 @@ ENGINE="vllm"
 VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}"
 VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
 
-# Benchmark scenarios: "concurrency input_len output_len".
+# Benchmark scenarios: each element is "concurrency input_len output_len".
-# Override via SCENARIOS env var.
+# Override via SCENARIOS env var as a bash array, e.g.:
-SCENARIOS="${SCENARIOS:-32 512 256   128 512 256   32 4000 512}"
+#   SCENARIOS=("32 512 256" "128 512 256")
+if [[ -z "${SCENARIOS:-}" ]]; then
+  SCENARIOS=(
+    "32 512 256"
+    "128 512 256"
+    "32 4000 512"
+  )
+fi
 NUM_PROMPTS="${NUM_PROMPTS:-128}"
 
 # Server start script bundled with this experiment.

experiments/dsv4_h200_vllm/parse_results.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 """Parse H200 vLLM baseline benchmark JSONL outputs.
 
-Reads raw JSONL files produced by `sglang.bench_serving --output-file --output-details`
+Reads the single-line summary JSON produced by
-and generates:
+`sglang.bench_serving --output-file --output-details` and generates:
     - results.json  (appended scenarios)
     - report.md     (human-readable summary)
 
@@ -11,94 +11,68 @@ Usage:
 """
 
 import json
-import math
-import os
 import sys
 from pathlib import Path
 
 
-def percentile(values: list[float], p: float) -> float:
+def parse_jsonl(path: Path) -> dict | None:
-    if not values:
+    """Read the first (and usually only) JSON object from the JSONL file."""
-        return 0.0
-    values = sorted(values)
-    if len(values) == 1:
-        return values[0]
-    k = (len(values) - 1) * (p / 100.0)
-    f = math.floor(k)
-    c = math.ceil(k)
-    if f == c:
-        return values[int(k)]
-    return values[f] * (c - k) + values[c] * (k - f)
-
-
-def parse_jsonl(path: Path) -> list[dict]:
-    requests = []
     with open(path, "r", encoding="utf-8") as f:
         for line in f:
             line = line.strip()
             if not line:
                 continue
             try:
-                requests.append(json.loads(line))
+                return json.loads(line)
             except json.JSONDecodeError:
                 continue
-    return requests
+    return None
 
 
-def compute_metrics(requests: list[dict]) -> dict:
+def compute_metrics(data: dict) -> dict:
-    success_reqs = [r for r in requests if r.get("success", True)]
+    completed = data.get("completed", 0)
-    failed = len(requests) - len(success_reqs)
+    total = len(data.get("input_lens", []))
-
+    failed = total - completed if total > 0 else 0
-    if not success_reqs:
+    duration_s = data.get("duration", 0.0)
-        return {"success": 0, "failed": failed}
-
-    # Time boundaries.
-    start_times = [r["tstamp_start"] for r in success_reqs]
-    end_times = [r["tstamp_finished"] for r in success_reqs]
-    duration_s = max(end_times) - min(start_times)
-
-    # Token counts.
-    input_tokens = [r.get("prompt_tokens", 0) for r in success_reqs]
-    output_tokens = [r.get("completion_tokens", 0) for r in success_reqs]
-    total_input = sum(input_tokens)
-    total_output = sum(output_tokens)
-
-    # Latencies (ms).
-    e2e = [r.get("e2e_latency", 0) * 1000 for r in success_reqs]
-    ttft = [r.get("ttft", 0) * 1000 for r in success_reqs]
-    itls = []
-    for r in success_reqs:
-        itls.extend(r.get("itl", []))
-    # TPOT from itl averages per request.
-    tpots = []
-    for r in success_reqs:
-        req_itls = r.get("itl", [])
-        if req_itls:
-            tpots.append(sum(req_itls) / len(req_itls) * 1000)
-
-    def latency_stats(values: list[float]) -> dict:
-        return {
-            "mean": sum(values) / len(values),
-            "p50": percentile(values, 50),
-            "p90": percentile(values, 90),
-            "p95": percentile(values, 95),
-            "p99": percentile(values, 99),
-        }
 
     return {
-        "success": len(success_reqs),
+        "success": completed,
         "failed": failed,
         "duration_s": duration_s,
-        "request_throughput": len(success_reqs) / duration_s if duration_s > 0 else 0.0,
+        "request_throughput": data.get("request_throughput", 0.0),
-        "input_token_throughput": total_input / duration_s if duration_s > 0 else 0.0,
+        "input_token_throughput": data.get("input_throughput", 0.0),
-        "output_token_throughput": total_output / duration_s if duration_s > 0 else 0.0,
+        "output_token_throughput": data.get("output_throughput", 0.0),
-        "total_token_throughput": (total_input + total_output) / duration_s if duration_s > 0 else 0.0,
+        "total_token_throughput": data.get("total_throughput", 0.0),
-        "total_input_tokens": total_input,
+        "total_input_tokens": data.get("total_input_tokens", 0),
-        "total_output_tokens": total_output,
+        "total_output_tokens": data.get("total_output_tokens", 0),
-        "e2e_ms": latency_stats(e2e),
+        "e2e_ms": {
-        "ttft_ms": latency_stats(ttft),
+            "mean": data.get("mean_e2e_latency_ms", 0.0),
-        "tpot_ms": latency_stats(tpots),
+            "p50": data.get("median_e2e_latency_ms", 0.0),
-        "itl_ms": latency_stats([v * 1000 for v in itls]),
+            "p90": data.get("p90_e2e_latency_ms", 0.0),
+            "p95": data.get("p95_e2e_latency_ms", 0.0),
+            "p99": data.get("p99_e2e_latency_ms", 0.0),
+        },
+        "ttft_ms": {
+            "mean": data.get("mean_ttft_ms", 0.0),
+            "p50": data.get("median_ttft_ms", 0.0),
+            "p90": data.get("p90_ttft_ms", 0.0),
+            "p95": data.get("p95_ttft_ms", 0.0),
+            "p99": data.get("p99_ttft_ms", 0.0),
+        },
+        "tpot_ms": {
+            "mean": data.get("mean_tpot_ms", 0.0),
+            "p50": data.get("median_tpot_ms", 0.0),
+            "p90": data.get("p90_tpot_ms", 0.0),
+            "p95": data.get("p95_tpot_ms", 0.0),
+            "p99": data.get("p99_tpot_ms", 0.0),
+        },
+        "itl_ms": {
+            "mean": data.get("mean_itl_ms", 0.0),
+            "p50": data.get("median_itl_ms", 0.0),
+            "p90": data.get("p90_itl_ms", 0.0),
+            "p95": data.get("p95_itl_ms", 0.0),
+            "p99": data.get("p99_itl_ms", 0.0),
+        },
     }
 
 
@@ -120,7 +94,7 @@ def generate_report(result_root: Path, scenarios: list[dict]) -> None:
         f.write("# H200 vLLM Baseline Benchmark Report\n\n")
         f.write(f"- Result root: `{result_root}`\n")
         f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
-        f.write("- Backend: vLLM (TP=8, FP8 KV cache, no speculative decoding)\n")
+        f.write("- Backend: vLLM (TP=4, FP8 KV cache, no speculative decoding)\n")
         f.write("- Benchmark client: `sglang.bench_serving --backend vllm`\n\n")
 
         f.write("## Results\n\n")
@@ -131,7 +105,7 @@ def generate_report(result_root: Path, scenarios: list[dict]) -> None:
             cfg = s["config"]
             m = s["metrics"]
             f.write(
-                f"| {cfg['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
+                f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
                 f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
                 f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
                 f"{m['total_token_throughput']:.2f} | "
@@ -158,11 +132,11 @@ def main() -> None:
             continue
         concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
 
-        requests = parse_jsonl(jsonl_path)
+        data = parse_jsonl(jsonl_path)
-        if not requests:
+        if data is None:
             continue
 
-        metrics = compute_metrics(requests)
+        metrics = compute_metrics(data)
         scenario = {
             "name": scenario_name(concurrency, input_len, output_len),
             "config": {

experiments/dsv4_h200_vllm/results/20260708-062348/report.md

@@ -0,0 +1,15 @@
+# H200 vLLM Baseline Benchmark Report
+
+- Result root: `experiments/dsv4_h200_vllm/results/20260708-062348`
+- Model: `/data/models/DeepSeek-V4-Flash`
+- Backend: vLLM (TP=4, FP8 KV cache, no speculative decoding)
+- Benchmark client: `sglang.bench_serving --backend vllm`
+
+## Results
+
+| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| c128_i512_o256 | 128 | 512 | 256 | 5.98 | 128 | 21.40 | 5892.52 | 2825.23 | 8717.75 | 634.93 | 873.62 | 879.97 | 28.09 | 40.67 | 91.17 | 3786.22 | 5857.90 | 5919.94 |
+| c32_i4000_o512 | 32 | 4000 | 512 | 63.97 | 128 | 2.00 | 4229.98 | 505.98 | 4735.95 | 917.05 | 2853.31 | 6638.21 | 62.67 | 165.57 | 219.72 | 15397.36 | 42952.86 | 46968.37 |
+| c32_i512_o256 | 32 | 512 | 256 | 38.26 | 128 | 3.35 | 921.37 | 441.76 | 1363.13 | 1228.47 | 11441.48 | 11446.53 | 79.50 | 246.70 | 674.49 | 9197.96 | 27486.69 | 28020.43 |
+

experiments/dsv4_h200_vllm/results/20260708-062348/results.json

@@ -0,0 +1,183 @@
+{
+  "metadata": {
+    "experiment": "dsv4_h200_vllm",
+    "run_id": "20260708-062348",
+    "timestamp": "2026-07-08T06:23:48+00:00",
+    "model": "/data/models/DeepSeek-V4-Flash",
+    "backend": "vllm",
+    "engine": "vllm",
+    "hardware": "8x NVIDIA H200 143GB",
+    "accelerator": "NVIDIA H200",
+    "chip": "nvidia_h200",
+    "script": "experiments/dsv4_h200_vllm/run_bench.sh",
+    "env": "/data/user1/yy/envs/vllm",
+    "git_commit": "acf2e3d",
+    "git_dirty": "dirty",
+    "description": "H200 native vLLM baseline benchmark for DeepSeek-V4-Flash"
+  },
+  "config": {
+    "tp": 4,
+    "kv_cache_dtype": "fp8",
+    "block_size": 256,
+    "max_num_seqs": 256,
+    "port": 30005,
+    "num_prompts": 128,
+    "scenarios": [
+      "32 512 256",
+      "128 512 256",
+      "32 4000 512"
+    ]
+  },
+  "scenarios": [
+    {
+      "name": "c128_i512_o256",
+      "config": {
+        "concurrency": 128,
+        "input_len": 512,
+        "output_len": 256,
+        "dataset": "random",
+        "num_prompts": 128
+      },
+      "metrics": {
+        "success": 128,
+        "failed": 0,
+        "duration_s": 5.98181911901338,
+        "request_throughput": 21.39817293925662,
+        "input_token_throughput": 5892.521873147793,
+        "output_token_throughput": 2825.2275208862256,
+        "total_token_throughput": 8717.749394034017,
+        "total_input_tokens": 35248,
+        "total_output_tokens": 16900,
+        "e2e_ms": {
+          "mean": 3786.2239771795885,
+          "p50": 3966.506554497755,
+          "p90": 5763.936419402307,
+          "p95": 5857.902326301701,
+          "p99": 5919.940241662553
+        },
+        "ttft_ms": {
+          "mean": 634.9285282819892,
+          "p50": 651.9191234983737,
+          "p90": 872.5053475049208,
+          "p95": 873.6197502956202,
+          "p99": 879.9700354097877
+        },
+        "tpot_ms": {
+          "mean": 28.086932902452045,
+          "p50": 24.940565250290092,
+          "p90": 33.6093050082046,
+          "p95": 40.66724980024446,
+          "p99": 91.16725345989823
+        },
+        "itl_ms": {
+          "mean": 24.055493866406856,
+          "p50": 22.871914501592983,
+          "p90": 26.806968497112393,
+          "p95": 27.607820050616283,
+          "p99": 32.99604622545303
+        }
+      },
+      "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_128_512_256.jsonl"
+    },
+    {
+      "name": "c32_i4000_o512",
+      "config": {
+        "concurrency": 32,
+        "input_len": 4000,
+        "output_len": 512,
+        "dataset": "random",
+        "num_prompts": 128
+      },
+      "metrics": {
+        "success": 128,
+        "failed": 0,
+        "duration_s": 63.96937208699819,
+        "request_throughput": 2.000957580542143,
+        "input_token_throughput": 4229.977427822796,
+        "output_token_throughput": 505.9765156984964,
+        "total_token_throughput": 4735.953943521293,
+        "total_input_tokens": 270589,
+        "total_output_tokens": 32367,
+        "e2e_ms": {
+          "mean": 15397.358620969157,
+          "p50": 10554.476282501128,
+          "p90": 39473.02784240455,
+          "p95": 42952.8573812,
+          "p99": 46968.37465279941
+        },
+        "ttft_ms": {
+          "mean": 917.0502477351192,
+          "p50": 236.0078425044776,
+          "p90": 2404.053003701846,
+          "p95": 2853.309046203503,
+          "p99": 6638.214706212312
+        },
+        "tpot_ms": {
+          "mean": 62.674261138657926,
+          "p50": 38.76617610605361,
+          "p90": 141.09892115795085,
+          "p95": 165.57312805125648,
+          "p99": 219.71921567381835
+        },
+        "itl_ms": {
+          "mean": 57.50244012726164,
+          "p50": 14.738622994627804,
+          "p90": 19.66081059363205,
+          "p95": 142.28972980054095,
+          "p99": 1669.9934365210356
+        }
+      },
+      "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_4000_512.jsonl"
+    },
+    {
+      "name": "c32_i512_o256",
+      "config": {
+        "concurrency": 32,
+        "input_len": 512,
+        "output_len": 256,
+        "dataset": "random",
+        "num_prompts": 128
+      },
+      "metrics": {
+        "success": 128,
+        "failed": 0,
+        "duration_s": 38.25599173500086,
+        "request_throughput": 3.3458811076355204,
+        "input_token_throughput": 921.3720100151314,
+        "output_token_throughput": 441.7608649925023,
+        "total_token_throughput": 1363.1328750076339,
+        "total_input_tokens": 35248,
+        "total_output_tokens": 16900,
+        "e2e_ms": {
+          "mean": 9197.956188436932,
+          "p50": 6626.5614014992025,
+          "p90": 20719.612499210045,
+          "p95": 27486.685669697905,
+          "p99": 28020.42960330684
+        },
+        "ttft_ms": {
+          "mean": 1228.4664978981255,
+          "p50": 132.46312249975745,
+          "p90": 3753.0612517031777,
+          "p95": 11441.480474699347,
+          "p99": 11446.530493390019
+        },
+        "tpot_ms": {
+          "mean": 79.49513794690715,
+          "p50": 35.04621934244705,
+          "p90": 158.92302877744768,
+          "p95": 246.69572046122963,
+          "p99": 674.493606643706
+        },
+        "itl_ms": {
+          "mean": 60.85736878373586,
+          "p50": 15.390933993330691,
+          "p90": 47.24743370315991,
+          "p95": 57.582202005869476,
+          "p99": 237.64244167061403
+        }
+      },
+      "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_512_256.jsonl"
+    }
+  ]
+}

experiments/dsv4_h200_vllm/run_bench.sh

@@ -57,13 +57,13 @@ with open(path, "r", encoding="utf-8") as f:
     data = json.load(f)
 
 data["config"] = {
-    "tp": 8,
+    "tp": 4,
     "kv_cache_dtype": "fp8",
     "block_size": 256,
     "max_num_seqs": 256,
     "port": 30005,
     "num_prompts": 128,
-    "scenarios": "32 512 256   128 512 256   32 4000 512"
+    "scenarios": ["32 512 256", "128 512 256", "32 4000 512"]
 }
 with open(path, "w", encoding="utf-8") as f:
     json.dump(data, f, indent=2, ensure_ascii=False)
@@ -127,7 +127,7 @@ fi
 
 log "===== BENCHMARK START ====="
 
-for scenario in ${SCENARIOS}; do
+for scenario in "${SCENARIOS[@]}"; do
   read -r concurrency input_len output_len <<< "$scenario"
   output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
   detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"