qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dsv4_h200_vllm): scenario array parsing and sglang output format

Browse Source 
- Use bash array for SCENARIOS to avoid word-splitting
- Fix metadata config (tp=4, scenarios as array)
- Rewrite parse_results.py for sglang.bench_serving --output-details format
- Update .gitignore to keep experiments/*/results/*.json and *.md,
  ignore only raw_outputs/ and logs/ subdirs
- First successful run: 20260708-062348
This commit is contained in:
Quantong Qiu 2026-07-08 06:31:03 +00:00
parent 8e15ffe1c7
commit b742187498
6 changed files with 265 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -23,8 +23,9 @@ datasets/
# 原始请求级输出（方案 A：极简版，不存原始 jsonl）
bench_results/**/raw_outputs/
# 实验级原始结果目录（report.md / results.json 可单独保留，默认忽略整个目录）
experiments/*/results/
# 实验级原始结果目录：保留 report.md / results.json，忽略原始 jsonl 和日志
experiments/*/results/raw_outputs/
experiments/*/results/logs/
# 无关项目
loomeval_yy/

13
experiments/dsv4_h200_vllm/config.env
View File

@ -15,9 +15,16 @@ ENGINE="vllm"
VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}"
VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
# Benchmark scenarios: "concurrency input_len output_len".
# Override via SCENARIOS env var.
SCENARIOS="${SCENARIOS:-32 512 256 128 512 256 32 4000 512}"
# Benchmark scenarios: each element is "concurrency input_len output_len".
# Override via SCENARIOS env var as a bash array, e.g.:
# SCENARIOS=("32 512 256" "128 512 256")
if [[ -z "${SCENARIOS:-}" ]]; then
SCENARIOS=(
"32 512 256"
"128 512 256"
"32 4000 512"
)
fi
NUM_PROMPTS="${NUM_PROMPTS:-128}"
# Server start script bundled with this experiment.

128
experiments/dsv4_h200_vllm/parse_results.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python3
"""Parse H200 vLLM baseline benchmark JSONL outputs.
Reads raw JSONL files produced by `sglang.bench_serving --output-file --output-details`
and generates:
Reads the single-line summary JSON produced by
`sglang.bench_serving --output-file --output-details` and generates:
- results.json (appended scenarios)
- report.md (human-readable summary)
@ -11,94 +11,68 @@ Usage:
"""
import json
import math
import os
import sys
from pathlib import Path
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
values = sorted(values)
if len(values) == 1:
return values[0]
k = (len(values) - 1) * (p / 100.0)
f = math.floor(k)
c = math.ceil(k)
if f == c:
return values[int(k)]
return values[f] * (c - k) + values[c] * (k - f)
def parse_jsonl(path: Path) -> list[dict]:
requests = []
def parse_jsonl(path: Path) -> dict | None:
"""Read the first (and usually only) JSON object from the JSONL file."""
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
requests.append(json.loads(line))
return json.loads(line)
except json.JSONDecodeError:
continue
return requests
return None
def compute_metrics(requests: list[dict]) -> dict:
success_reqs = [r for r in requests if r.get("success", True)]
failed = len(requests) - len(success_reqs)
if not success_reqs:
return {"success": 0, "failed": failed}
# Time boundaries.
start_times = [r["tstamp_start"] for r in success_reqs]
end_times = [r["tstamp_finished"] for r in success_reqs]
duration_s = max(end_times) - min(start_times)
# Token counts.
input_tokens = [r.get("prompt_tokens", 0) for r in success_reqs]
output_tokens = [r.get("completion_tokens", 0) for r in success_reqs]
total_input = sum(input_tokens)
total_output = sum(output_tokens)
# Latencies (ms).
e2e = [r.get("e2e_latency", 0) * 1000 for r in success_reqs]
ttft = [r.get("ttft", 0) * 1000 for r in success_reqs]
itls = []
for r in success_reqs:
itls.extend(r.get("itl", []))
# TPOT from itl averages per request.
tpots = []
for r in success_reqs:
req_itls = r.get("itl", [])
if req_itls:
tpots.append(sum(req_itls) / len(req_itls) * 1000)
def latency_stats(values: list[float]) -> dict:
return {
"mean": sum(values) / len(values),
"p50": percentile(values, 50),
"p90": percentile(values, 90),
"p95": percentile(values, 95),
"p99": percentile(values, 99),
}
def compute_metrics(data: dict) -> dict:
completed = data.get("completed", 0)
total = len(data.get("input_lens", []))
failed = total - completed if total > 0 else 0
duration_s = data.get("duration", 0.0)
return {
"success": len(success_reqs),
"success": completed,
"failed": failed,
"duration_s": duration_s,
"request_throughput": len(success_reqs) / duration_s if duration_s > 0 else 0.0,
"input_token_throughput": total_input / duration_s if duration_s > 0 else 0.0,
"output_token_throughput": total_output / duration_s if duration_s > 0 else 0.0,
"total_token_throughput": (total_input + total_output) / duration_s if duration_s > 0 else 0.0,
"total_input_tokens": total_input,
"total_output_tokens": total_output,
"e2e_ms": latency_stats(e2e),
"ttft_ms": latency_stats(ttft),
"tpot_ms": latency_stats(tpots),
"itl_ms": latency_stats([v * 1000 for v in itls]),
"request_throughput": data.get("request_throughput", 0.0),
"input_token_throughput": data.get("input_throughput", 0.0),
"output_token_throughput": data.get("output_throughput", 0.0),
"total_token_throughput": data.get("total_throughput", 0.0),
"total_input_tokens": data.get("total_input_tokens", 0),
"total_output_tokens": data.get("total_output_tokens", 0),
"e2e_ms": {
"mean": data.get("mean_e2e_latency_ms", 0.0),
"p50": data.get("median_e2e_latency_ms", 0.0),
"p90": data.get("p90_e2e_latency_ms", 0.0),
"p95": data.get("p95_e2e_latency_ms", 0.0),
"p99": data.get("p99_e2e_latency_ms", 0.0),
},
"ttft_ms": {
"mean": data.get("mean_ttft_ms", 0.0),
"p50": data.get("median_ttft_ms", 0.0),
"p90": data.get("p90_ttft_ms", 0.0),
"p95": data.get("p95_ttft_ms", 0.0),
"p99": data.get("p99_ttft_ms", 0.0),
},
"tpot_ms": {
"mean": data.get("mean_tpot_ms", 0.0),
"p50": data.get("median_tpot_ms", 0.0),
"p90": data.get("p90_tpot_ms", 0.0),
"p95": data.get("p95_tpot_ms", 0.0),
"p99": data.get("p99_tpot_ms", 0.0),
},
"itl_ms": {
"mean": data.get("mean_itl_ms", 0.0),
"p50": data.get("median_itl_ms", 0.0),
"p90": data.get("p90_itl_ms", 0.0),
"p95": data.get("p95_itl_ms", 0.0),
"p99": data.get("p99_itl_ms", 0.0),
},
}
@ -120,7 +94,7 @@ def generate_report(result_root: Path, scenarios: list[dict]) -> None:
f.write("# H200 vLLM Baseline Benchmark Report\n\n")
f.write(f"- Result root: `{result_root}`\n")
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
f.write("- Backend: vLLM (TP=8, FP8 KV cache, no speculative decoding)\n")
f.write("- Backend: vLLM (TP=4, FP8 KV cache, no speculative decoding)\n")
f.write("- Benchmark client: `sglang.bench_serving --backend vllm`\n\n")
f.write("## Results\n\n")
@ -131,7 +105,7 @@ def generate_report(result_root: Path, scenarios: list[dict]) -> None:
cfg = s["config"]
m = s["metrics"]
f.write(
f"| {cfg['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['total_token_throughput']:.2f} | "
@ -158,11 +132,11 @@ def main() -> None:
continue
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
requests = parse_jsonl(jsonl_path)
if not requests:
data = parse_jsonl(jsonl_path)
if data is None:
continue
metrics = compute_metrics(requests)
metrics = compute_metrics(data)
scenario = {
"name": scenario_name(concurrency, input_len, output_len),
"config": {

15
experiments/dsv4_h200_vllm/results/20260708-062348/report.md Normal file
View File

@ -0,0 +1,15 @@
# H200 vLLM Baseline Benchmark Report
- Result root: `experiments/dsv4_h200_vllm/results/20260708-062348`
- Model: `/data/models/DeepSeek-V4-Flash`
- Backend: vLLM (TP=4, FP8 KV cache, no speculative decoding)
- Benchmark client: `sglang.bench_serving --backend vllm`
## Results
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
| c128_i512_o256 | 128 | 512 | 256 | 5.98 | 128 | 21.40 | 5892.52 | 2825.23 | 8717.75 | 634.93 | 873.62 | 879.97 | 28.09 | 40.67 | 91.17 | 3786.22 | 5857.90 | 5919.94 |
| c32_i4000_o512 | 32 | 4000 | 512 | 63.97 | 128 | 2.00 | 4229.98 | 505.98 | 4735.95 | 917.05 | 2853.31 | 6638.21 | 62.67 | 165.57 | 219.72 | 15397.36 | 42952.86 | 46968.37 |
| c32_i512_o256 | 32 | 512 | 256 | 38.26 | 128 | 3.35 | 921.37 | 441.76 | 1363.13 | 1228.47 | 11441.48 | 11446.53 | 79.50 | 246.70 | 674.49 | 9197.96 | 27486.69 | 28020.43 |

183
experiments/dsv4_h200_vllm/results/20260708-062348/results.json Normal file
View File

@ -0,0 +1,183 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm",
"run_id": "20260708-062348",
"timestamp": "2026-07-08T06:23:48+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
"hardware": "8x NVIDIA H200 143GB",
"accelerator": "NVIDIA H200",
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "acf2e3d",
"git_dirty": "dirty",
"description": "H200 native vLLM baseline benchmark for DeepSeek-V4-Flash"
},
"config": {
"tp": 4,
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"port": 30005,
"num_prompts": 128,
"scenarios": [
"32 512 256",
"128 512 256",
"32 4000 512"
]
},
"scenarios": [
{
"name": "c128_i512_o256",
"config": {
"concurrency": 128,
"input_len": 512,
"output_len": 256,
"dataset": "random",
"num_prompts": 128
},
"metrics": {
"success": 128,
"failed": 0,
"duration_s": 5.98181911901338,
"request_throughput": 21.39817293925662,
"input_token_throughput": 5892.521873147793,
"output_token_throughput": 2825.2275208862256,
"total_token_throughput": 8717.749394034017,
"total_input_tokens": 35248,
"total_output_tokens": 16900,
"e2e_ms": {
"mean": 3786.2239771795885,
"p50": 3966.506554497755,
"p90": 5763.936419402307,
"p95": 5857.902326301701,
"p99": 5919.940241662553
},
"ttft_ms": {
"mean": 634.9285282819892,
"p50": 651.9191234983737,
"p90": 872.5053475049208,
"p95": 873.6197502956202,
"p99": 879.9700354097877
},
"tpot_ms": {
"mean": 28.086932902452045,
"p50": 24.940565250290092,
"p90": 33.6093050082046,
"p95": 40.66724980024446,
"p99": 91.16725345989823
},
"itl_ms": {
"mean": 24.055493866406856,
"p50": 22.871914501592983,
"p90": 26.806968497112393,
"p95": 27.607820050616283,
"p99": 32.99604622545303
}
},
"raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_128_512_256.jsonl"
},
{
"name": "c32_i4000_o512",
"config": {
"concurrency": 32,
"input_len": 4000,
"output_len": 512,
"dataset": "random",
"num_prompts": 128
},
"metrics": {
"success": 128,
"failed": 0,
"duration_s": 63.96937208699819,
"request_throughput": 2.000957580542143,
"input_token_throughput": 4229.977427822796,
"output_token_throughput": 505.9765156984964,
"total_token_throughput": 4735.953943521293,
"total_input_tokens": 270589,
"total_output_tokens": 32367,
"e2e_ms": {
"mean": 15397.358620969157,
"p50": 10554.476282501128,
"p90": 39473.02784240455,
"p95": 42952.8573812,
"p99": 46968.37465279941
},
"ttft_ms": {
"mean": 917.0502477351192,
"p50": 236.0078425044776,
"p90": 2404.053003701846,
"p95": 2853.309046203503,
"p99": 6638.214706212312
},
"tpot_ms": {
"mean": 62.674261138657926,
"p50": 38.76617610605361,
"p90": 141.09892115795085,
"p95": 165.57312805125648,
"p99": 219.71921567381835
},
"itl_ms": {
"mean": 57.50244012726164,
"p50": 14.738622994627804,
"p90": 19.66081059363205,
"p95": 142.28972980054095,
"p99": 1669.9934365210356
}
},
"raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_4000_512.jsonl"
},
{
"name": "c32_i512_o256",
"config": {
"concurrency": 32,
"input_len": 512,
"output_len": 256,
"dataset": "random",
"num_prompts": 128
},
"metrics": {
"success": 128,
"failed": 0,
"duration_s": 38.25599173500086,
"request_throughput": 3.3458811076355204,
"input_token_throughput": 921.3720100151314,
"output_token_throughput": 441.7608649925023,
"total_token_throughput": 1363.1328750076339,
"total_input_tokens": 35248,
"total_output_tokens": 16900,
"e2e_ms": {
"mean": 9197.956188436932,
"p50": 6626.5614014992025,
"p90": 20719.612499210045,
"p95": 27486.685669697905,
"p99": 28020.42960330684
},
"ttft_ms": {
"mean": 1228.4664978981255,
"p50": 132.46312249975745,
"p90": 3753.0612517031777,
"p95": 11441.480474699347,
"p99": 11446.530493390019
},
"tpot_ms": {
"mean": 79.49513794690715,
"p50": 35.04621934244705,
"p90": 158.92302877744768,
"p95": 246.69572046122963,
"p99": 674.493606643706
},
"itl_ms": {
"mean": 60.85736878373586,
"p50": 15.390933993330691,
"p90": 47.24743370315991,
"p95": 57.582202005869476,
"p99": 237.64244167061403
}
},
"raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_512_256.jsonl"
}
]
}

6
experiments/dsv4_h200_vllm/run_bench.sh
View File

@ -57,13 +57,13 @@ with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
data["config"] = {
"tp": 8,
"tp": 4,
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"port": 30005,
"num_prompts": 128,
"scenarios": "32 512 256 128 512 256 32 4000 512"
"scenarios": ["32 512 256", "128 512 256", "32 4000 512"]
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
@ -127,7 +127,7 @@ fi
log "===== BENCHMARK START ====="
for scenario in ${SCENARIOS}; do
for scenario in "${SCENARIOS[@]}"; do
read -r concurrency input_len output_len <<< "$scenario"
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"