From b742187498d2eaad42f7b6f6f545690312183e41 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Wed, 8 Jul 2026 06:31:03 +0000 Subject: [PATCH] fix(dsv4_h200_vllm): scenario array parsing and sglang output format - Use bash array for SCENARIOS to avoid word-splitting - Fix metadata config (tp=4, scenarios as array) - Rewrite parse_results.py for sglang.bench_serving --output-details format - Update .gitignore to keep experiments/*/results/*.json and *.md, ignore only raw_outputs/ and logs/ subdirs - First successful run: 20260708-062348 --- .gitignore | 5 +- experiments/dsv4_h200_vllm/config.env | 13 +- experiments/dsv4_h200_vllm/parse_results.py | 128 +++++------- .../results/20260708-062348/report.md | 15 ++ .../results/20260708-062348/results.json | 183 ++++++++++++++++++ experiments/dsv4_h200_vllm/run_bench.sh | 6 +- 6 files changed, 265 insertions(+), 85 deletions(-) create mode 100644 experiments/dsv4_h200_vllm/results/20260708-062348/report.md create mode 100644 experiments/dsv4_h200_vllm/results/20260708-062348/results.json diff --git a/.gitignore b/.gitignore index 43815d7..0b81dda 100644 --- a/.gitignore +++ b/.gitignore @@ -23,8 +23,9 @@ datasets/ # 原始请求级输出（方案 A：极简版，不存原始 jsonl） bench_results/**/raw_outputs/ -# 实验级原始结果目录（report.md / results.json 可单独保留，默认忽略整个目录） -experiments/*/results/ +# 实验级原始结果目录：保留 report.md / results.json，忽略原始 jsonl 和日志 +experiments/*/results/raw_outputs/ +experiments/*/results/logs/ # 无关项目 loomeval_yy/ diff --git a/experiments/dsv4_h200_vllm/config.env b/experiments/dsv4_h200_vllm/config.env index 1a00668..cda9047 100644 --- a/experiments/dsv4_h200_vllm/config.env +++ b/experiments/dsv4_h200_vllm/config.env @@ -15,9 +15,16 @@ ENGINE="vllm" VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}" VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# Benchmark scenarios: "concurrency input_len output_len". -# Override via SCENARIOS env var. -SCENARIOS="${SCENARIOS:-32 512 256 128 512 256 32 4000 512}" +# Benchmark scenarios: each element is "concurrency input_len output_len". +# Override via SCENARIOS env var as a bash array, e.g.: +# SCENARIOS=("32 512 256" "128 512 256") +if [[ -z "${SCENARIOS:-}" ]]; then + SCENARIOS=( + "32 512 256" + "128 512 256" + "32 4000 512" + ) +fi NUM_PROMPTS="${NUM_PROMPTS:-128}" # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm/parse_results.py b/experiments/dsv4_h200_vllm/parse_results.py index 642065b..ebb4e1a 100755 --- a/experiments/dsv4_h200_vllm/parse_results.py +++ b/experiments/dsv4_h200_vllm/parse_results.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Parse H200 vLLM baseline benchmark JSONL outputs. -Reads raw JSONL files produced by `sglang.bench_serving --output-file --output-details` -and generates: +Reads the single-line summary JSON produced by +`sglang.bench_serving --output-file --output-details` and generates: - results.json (appended scenarios) - report.md (human-readable summary) @@ -11,94 +11,68 @@ Usage: """ import json -import math -import os import sys from pathlib import Path -def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - values = sorted(values) - if len(values) == 1: - return values[0] - k = (len(values) - 1) * (p / 100.0) - f = math.floor(k) - c = math.ceil(k) - if f == c: - return values[int(k)] - return values[f] * (c - k) + values[c] * (k - f) - - -def parse_jsonl(path: Path) -> list[dict]: - requests = [] +def parse_jsonl(path: Path) -> dict | None: + """Read the first (and usually only) JSON object from the JSONL file.""" with open(path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: - requests.append(json.loads(line)) + return json.loads(line) except json.JSONDecodeError: continue - return requests + return None -def compute_metrics(requests: list[dict]) -> dict: - success_reqs = [r for r in requests if r.get("success", True)] - failed = len(requests) - len(success_reqs) - - if not success_reqs: - return {"success": 0, "failed": failed} - - # Time boundaries. - start_times = [r["tstamp_start"] for r in success_reqs] - end_times = [r["tstamp_finished"] for r in success_reqs] - duration_s = max(end_times) - min(start_times) - - # Token counts. - input_tokens = [r.get("prompt_tokens", 0) for r in success_reqs] - output_tokens = [r.get("completion_tokens", 0) for r in success_reqs] - total_input = sum(input_tokens) - total_output = sum(output_tokens) - - # Latencies (ms). - e2e = [r.get("e2e_latency", 0) * 1000 for r in success_reqs] - ttft = [r.get("ttft", 0) * 1000 for r in success_reqs] - itls = [] - for r in success_reqs: - itls.extend(r.get("itl", [])) - # TPOT from itl averages per request. - tpots = [] - for r in success_reqs: - req_itls = r.get("itl", []) - if req_itls: - tpots.append(sum(req_itls) / len(req_itls) * 1000) - - def latency_stats(values: list[float]) -> dict: - return { - "mean": sum(values) / len(values), - "p50": percentile(values, 50), - "p90": percentile(values, 90), - "p95": percentile(values, 95), - "p99": percentile(values, 99), - } +def compute_metrics(data: dict) -> dict: + completed = data.get("completed", 0) + total = len(data.get("input_lens", [])) + failed = total - completed if total > 0 else 0 + duration_s = data.get("duration", 0.0) return { - "success": len(success_reqs), + "success": completed, "failed": failed, "duration_s": duration_s, - "request_throughput": len(success_reqs) / duration_s if duration_s > 0 else 0.0, - "input_token_throughput": total_input / duration_s if duration_s > 0 else 0.0, - "output_token_throughput": total_output / duration_s if duration_s > 0 else 0.0, - "total_token_throughput": (total_input + total_output) / duration_s if duration_s > 0 else 0.0, - "total_input_tokens": total_input, - "total_output_tokens": total_output, - "e2e_ms": latency_stats(e2e), - "ttft_ms": latency_stats(ttft), - "tpot_ms": latency_stats(tpots), - "itl_ms": latency_stats([v * 1000 for v in itls]), + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_token_throughput": data.get("total_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, } @@ -120,7 +94,7 @@ def generate_report(result_root: Path, scenarios: list[dict]) -> None: f.write("# H200 vLLM Baseline Benchmark Report\n\n") f.write(f"- Result root: `{result_root}`\n") f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write("- Backend: vLLM (TP=8, FP8 KV cache, no speculative decoding)\n") + f.write("- Backend: vLLM (TP=4, FP8 KV cache, no speculative decoding)\n") f.write("- Benchmark client: `sglang.bench_serving --backend vllm`\n\n") f.write("## Results\n\n") @@ -131,7 +105,7 @@ def generate_report(result_root: Path, scenarios: list[dict]) -> None: cfg = s["config"] m = s["metrics"] f.write( - f"| {cfg['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " f"{m['total_token_throughput']:.2f} | " @@ -158,11 +132,11 @@ def main() -> None: continue concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - requests = parse_jsonl(jsonl_path) - if not requests: + data = parse_jsonl(jsonl_path) + if data is None: continue - metrics = compute_metrics(requests) + metrics = compute_metrics(data) scenario = { "name": scenario_name(concurrency, input_len, output_len), "config": { diff --git a/experiments/dsv4_h200_vllm/results/20260708-062348/report.md b/experiments/dsv4_h200_vllm/results/20260708-062348/report.md new file mode 100644 index 0000000..ef2b66e --- /dev/null +++ b/experiments/dsv4_h200_vllm/results/20260708-062348/report.md @@ -0,0 +1,15 @@ +# H200 vLLM Baseline Benchmark Report + +- Result root: `experiments/dsv4_h200_vllm/results/20260708-062348` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: vLLM (TP=4, FP8 KV cache, no speculative decoding) +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | 128 | 512 | 256 | 5.98 | 128 | 21.40 | 5892.52 | 2825.23 | 8717.75 | 634.93 | 873.62 | 879.97 | 28.09 | 40.67 | 91.17 | 3786.22 | 5857.90 | 5919.94 | +| c32_i4000_o512 | 32 | 4000 | 512 | 63.97 | 128 | 2.00 | 4229.98 | 505.98 | 4735.95 | 917.05 | 2853.31 | 6638.21 | 62.67 | 165.57 | 219.72 | 15397.36 | 42952.86 | 46968.37 | +| c32_i512_o256 | 32 | 512 | 256 | 38.26 | 128 | 3.35 | 921.37 | 441.76 | 1363.13 | 1228.47 | 11441.48 | 11446.53 | 79.50 | 246.70 | 674.49 | 9197.96 | 27486.69 | 28020.43 | + diff --git a/experiments/dsv4_h200_vllm/results/20260708-062348/results.json b/experiments/dsv4_h200_vllm/results/20260708-062348/results.json new file mode 100644 index 0000000..6d28b47 --- /dev/null +++ b/experiments/dsv4_h200_vllm/results/20260708-062348/results.json @@ -0,0 +1,183 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm", + "run_id": "20260708-062348", + "timestamp": "2026-07-08T06:23:48+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "acf2e3d", + "git_dirty": "dirty", + "description": "H200 native vLLM baseline benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 4, + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "port": 30005, + "num_prompts": 128, + "scenarios": [ + "32 512 256", + "128 512 256", + "32 4000 512" + ] + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 5.98181911901338, + "request_throughput": 21.39817293925662, + "input_token_throughput": 5892.521873147793, + "output_token_throughput": 2825.2275208862256, + "total_token_throughput": 8717.749394034017, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 3786.2239771795885, + "p50": 3966.506554497755, + "p90": 5763.936419402307, + "p95": 5857.902326301701, + "p99": 5919.940241662553 + }, + "ttft_ms": { + "mean": 634.9285282819892, + "p50": 651.9191234983737, + "p90": 872.5053475049208, + "p95": 873.6197502956202, + "p99": 879.9700354097877 + }, + "tpot_ms": { + "mean": 28.086932902452045, + "p50": 24.940565250290092, + "p90": 33.6093050082046, + "p95": 40.66724980024446, + "p99": 91.16725345989823 + }, + "itl_ms": { + "mean": 24.055493866406856, + "p50": 22.871914501592983, + "p90": 26.806968497112393, + "p95": 27.607820050616283, + "p99": 32.99604622545303 + } + }, + "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_128_512_256.jsonl" + }, + { + "name": "c32_i4000_o512", + "config": { + "concurrency": 32, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 63.96937208699819, + "request_throughput": 2.000957580542143, + "input_token_throughput": 4229.977427822796, + "output_token_throughput": 505.9765156984964, + "total_token_throughput": 4735.953943521293, + "total_input_tokens": 270589, + "total_output_tokens": 32367, + "e2e_ms": { + "mean": 15397.358620969157, + "p50": 10554.476282501128, + "p90": 39473.02784240455, + "p95": 42952.8573812, + "p99": 46968.37465279941 + }, + "ttft_ms": { + "mean": 917.0502477351192, + "p50": 236.0078425044776, + "p90": 2404.053003701846, + "p95": 2853.309046203503, + "p99": 6638.214706212312 + }, + "tpot_ms": { + "mean": 62.674261138657926, + "p50": 38.76617610605361, + "p90": 141.09892115795085, + "p95": 165.57312805125648, + "p99": 219.71921567381835 + }, + "itl_ms": { + "mean": 57.50244012726164, + "p50": 14.738622994627804, + "p90": 19.66081059363205, + "p95": 142.28972980054095, + "p99": 1669.9934365210356 + } + }, + "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_4000_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 38.25599173500086, + "request_throughput": 3.3458811076355204, + "input_token_throughput": 921.3720100151314, + "output_token_throughput": 441.7608649925023, + "total_token_throughput": 1363.1328750076339, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 9197.956188436932, + "p50": 6626.5614014992025, + "p90": 20719.612499210045, + "p95": 27486.685669697905, + "p99": 28020.42960330684 + }, + "ttft_ms": { + "mean": 1228.4664978981255, + "p50": 132.46312249975745, + "p90": 3753.0612517031777, + "p95": 11441.480474699347, + "p99": 11446.530493390019 + }, + "tpot_ms": { + "mean": 79.49513794690715, + "p50": 35.04621934244705, + "p90": 158.92302877744768, + "p95": 246.69572046122963, + "p99": 674.493606643706 + }, + "itl_ms": { + "mean": 60.85736878373586, + "p50": 15.390933993330691, + "p90": 47.24743370315991, + "p95": 57.582202005869476, + "p99": 237.64244167061403 + } + }, + "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_512_256.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm/run_bench.sh b/experiments/dsv4_h200_vllm/run_bench.sh index e31c556..12f5245 100755 --- a/experiments/dsv4_h200_vllm/run_bench.sh +++ b/experiments/dsv4_h200_vllm/run_bench.sh @@ -57,13 +57,13 @@ with open(path, "r", encoding="utf-8") as f: data = json.load(f) data["config"] = { - "tp": 8, + "tp": 4, "kv_cache_dtype": "fp8", "block_size": 256, "max_num_seqs": 256, "port": 30005, "num_prompts": 128, - "scenarios": "32 512 256 128 512 256 32 4000 512" + "scenarios": ["32 512 256", "128 512 256", "32 4000 512"] } with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) @@ -127,7 +127,7 @@ fi log "===== BENCHMARK START =====" -for scenario in ${SCENARIOS}; do +for scenario in "${SCENARIOS[@]}"; do read -r concurrency input_len output_len <<< "$scenario" output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"