results(long-context-matrix): complete 64k/128k/256k matrix run 20260708-100016; use common scripts
This commit is contained in:
yy-fighting
parent
9b135aa4ff
commit
27ceaac47f
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a side-by-side comparison of SGLang and vLLM results.
|
||||
|
||||
Usage:
|
||||
python3 compare.py --sglang <sglang_result_root> --vllm <vllm_result_root> \
|
||||
[--output comparison.md]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_result(result_root: Path) -> dict:
|
||||
path = result_root / "results.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
|
||||
# S2 tier targets from docs/SLO_STANDARDS.md.
|
||||
ttft_ok = ttft_p95_ms < 3000.0
|
||||
tpot_ok = tpot_mean_ms < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
return "✅"
|
||||
if ttft_ok or tpot_ok:
|
||||
return "⚠️"
|
||||
return "❌"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--sglang", type=Path, required=True)
|
||||
parser.add_argument("--vllm", type=Path, required=True)
|
||||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||
args = parser.parse_args()
|
||||
|
||||
sglang_data = load_result(args.sglang)
|
||||
vllm_data = load_result(args.vllm)
|
||||
|
||||
by_scenario = defaultdict(dict)
|
||||
for data in (sglang_data, vllm_data):
|
||||
backend = data["metadata"]["engine"]
|
||||
for s in data.get("scenarios", []):
|
||||
key = s["name"]
|
||||
by_scenario[key][backend] = s
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n")
|
||||
f.write("## Summary\n\n")
|
||||
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write("- Hardware: 8x NVIDIA H200 143GB\n")
|
||||
f.write("- Tensor Parallelism: 8\n")
|
||||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||
f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n")
|
||||
|
||||
f.write("## Side-by-side results\n\n")
|
||||
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
|
||||
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for scenario_name in sorted(by_scenario.keys()):
|
||||
for backend in ("sglang", "vllm"):
|
||||
s = by_scenario[scenario_name].get(backend)
|
||||
if s is None:
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"])
|
||||
f.write(
|
||||
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
|
||||
)
|
||||
|
||||
f.write("\n## Notes\n\n")
|
||||
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `docs/SLO_STANDARDS.md`).\n")
|
||||
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
||||
|
||||
print(f"Wrote comparison to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,219 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse raw sglang.bench_serving JSONL outputs for one backend.
|
||||
|
||||
Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl
|
||||
and generates results.json + report.md in the given result root.
|
||||
|
||||
Usage:
|
||||
python3 parse_backend.py <result_root> [--backend sglang|vllm]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_jsonl(path: Path) -> dict | None:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
return json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def compute_metrics(data: dict) -> dict:
|
||||
completed = data.get("completed", 0)
|
||||
total = len(data.get("input_lens", []))
|
||||
failed = total - completed if total > 0 else 0
|
||||
duration_s = data.get("duration", 0.0)
|
||||
|
||||
return {
|
||||
"success": completed,
|
||||
"failed": failed,
|
||||
"duration_s": duration_s,
|
||||
"request_throughput": data.get("request_throughput", 0.0),
|
||||
"input_token_throughput": data.get("input_throughput", 0.0),
|
||||
"output_token_throughput": data.get("output_throughput", 0.0),
|
||||
"total_token_throughput": data.get("total_throughput", 0.0),
|
||||
"total_input_tokens": data.get("total_input_tokens", 0),
|
||||
"total_output_tokens": data.get("total_output_tokens", 0),
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms", 0.0),
|
||||
"p50": data.get("median_e2e_latency_ms", 0.0),
|
||||
"p90": data.get("p90_e2e_latency_ms", 0.0),
|
||||
"p95": data.get("p95_e2e_latency_ms", 0.0),
|
||||
"p99": data.get("p99_e2e_latency_ms", 0.0),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms", 0.0),
|
||||
"p50": data.get("median_ttft_ms", 0.0),
|
||||
"p90": data.get("p90_ttft_ms", 0.0),
|
||||
"p95": data.get("p95_ttft_ms", 0.0),
|
||||
"p99": data.get("p99_ttft_ms", 0.0),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms", 0.0),
|
||||
"p50": data.get("median_tpot_ms", 0.0),
|
||||
"p90": data.get("p90_tpot_ms", 0.0),
|
||||
"p95": data.get("p95_tpot_ms", 0.0),
|
||||
"p99": data.get("p99_tpot_ms", 0.0),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms", 0.0),
|
||||
"p50": data.get("median_itl_ms", 0.0),
|
||||
"p90": data.get("p90_itl_ms", 0.0),
|
||||
"p95": data.get("p95_itl_ms", 0.0),
|
||||
"p99": data.get("p99_itl_ms", 0.0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
||||
|
||||
|
||||
def slo_status(metrics: dict) -> dict:
|
||||
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
|
||||
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
|
||||
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
mark = "✅"
|
||||
elif ttft_ok or tpot_ok:
|
||||
mark = "⚠️"
|
||||
else:
|
||||
mark = "❌"
|
||||
return {
|
||||
"ttft_p95_ok": ttft_ok,
|
||||
"tpot_mean_ok": tpot_ok,
|
||||
"overall": mark,
|
||||
}
|
||||
|
||||
|
||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"].append(scenario)
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None:
|
||||
report_path = result_root / "report.md"
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n")
|
||||
f.write(f"- Result root: `{result_root}`\n")
|
||||
f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write(f"- Backend: {backend.upper()} (TP=8)\n")
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("result_root", type=Path)
|
||||
parser.add_argument("--backend", default=None, choices=["sglang", "vllm"])
|
||||
args = parser.parse_args()
|
||||
|
||||
result_root = args.result_root
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
results_json = result_root / "results.json"
|
||||
|
||||
if not raw_dir.exists():
|
||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
||||
|
||||
backend = args.backend
|
||||
if backend is None:
|
||||
# Infer from filenames if not provided.
|
||||
for p in raw_dir.iterdir():
|
||||
if p.name.startswith("sglang_"):
|
||||
backend = "sglang"
|
||||
break
|
||||
if p.name.startswith("vllm_"):
|
||||
backend = "vllm"
|
||||
break
|
||||
if backend is None:
|
||||
raise SystemExit("Could not infer backend from raw outputs")
|
||||
|
||||
scenarios = []
|
||||
for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
|
||||
parts = jsonl_path.stem.split("_")
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
label = parts[1]
|
||||
if label == "sharegpt":
|
||||
phase = "sharegpt"
|
||||
dataset = "sharegpt"
|
||||
else:
|
||||
phase = label
|
||||
dataset = "random"
|
||||
|
||||
# The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
|
||||
try:
|
||||
concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
data = parse_jsonl(jsonl_path)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
metrics = compute_metrics(data)
|
||||
scenario = {
|
||||
"name": scenario_name(concurrency, input_len, output_len),
|
||||
"config": {
|
||||
"phase": phase,
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"dataset": dataset,
|
||||
"num_prompts": metrics["success"] + metrics["failed"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"slo_status": slo_status(metrics),
|
||||
"raw_file": str(jsonl_path),
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
if not scenarios:
|
||||
print("No benchmark outputs found to parse")
|
||||
return
|
||||
|
||||
if results_json.exists():
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"] = scenarios
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
generate_report(result_root, backend, scenarios)
|
||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,61 @@
|
||||
# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)
|
||||
|
||||
## Summary
|
||||
|
||||
- Model: `/data/models/DeepSeek-V4-Flash`
|
||||
- Hardware: 8x NVIDIA H200 143GB
|
||||
- Tensor Parallelism: 8
|
||||
- Benchmark client: `sglang.bench_serving`
|
||||
- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)
|
||||
|
||||
## Side-by-side results
|
||||
|
||||
| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |
|
||||
|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| c10_i131072_o1024 | sglang | 10 | 131072 | 1024 | 0.68 | 319.74 | 4147.51 | 7409.82 | 25.69 | 49.89 | 58.92 | 26432.66 | ⚠️ |
|
||||
| c10_i131072_o1024 | vllm | 10 | 131072 | 1024 | 0.39 | 183.08 | 6596.15 | 6603.12 | 47.10 | 77.53 | 84.06 | 48426.64 | ⚠️ |
|
||||
| c10_i131072_o256 | sglang | 10 | 131072 | 256 | 0.34 | 42.70 | 26046.62 | 26064.38 | 1374.50 | 1810.56 | 19553.62 | 58326.41 | ❌ |
|
||||
| c10_i131072_o256 | vllm | 10 | 131072 | 256 | 0.38 | 48.09 | 23250.03 | 24238.33 | 150.22 | 299.91 | 301.72 | 51439.08 | ❌ |
|
||||
| c10_i131072_o4096 | sglang | 10 | 131072 | 4096 | 0.27 | 517.96 | 6177.42 | 6920.28 | 14.21 | 19.92 | 25.87 | 54286.63 | ⚠️ |
|
||||
| c10_i131072_o4096 | vllm | 10 | 131072 | 4096 | 0.23 | 445.81 | 2096.29 | 2168.81 | 18.57 | 21.47 | 21.61 | 67041.13 | ✅ |
|
||||
| c10_i262144_o1024 | sglang | 10 | 262144 | 1024 | 0.16 | 76.30 | 39446.96 | 43965.67 | 108.42 | 202.43 | 202.90 | 120180.87 | ❌ |
|
||||
| c10_i262144_o1024 | vllm | 10 | 262144 | 1024 | 0.18 | 84.63 | 34130.22 | 38144.40 | 98.87 | 142.57 | 266.08 | 108658.89 | ❌ |
|
||||
| c10_i262144_o256 | sglang | 10 | 262144 | 256 | 0.15 | 18.70 | 58074.31 | 62652.59 | 2953.70 | 3625.59 | 41185.32 | 134628.33 | ❌ |
|
||||
| c10_i262144_o256 | vllm | 10 | 262144 | 256 | 0.18 | 22.76 | 50632.44 | 51883.02 | 229.20 | 339.27 | 344.95 | 110216.92 | ❌ |
|
||||
| c10_i262144_o4096 | sglang | 10 | 262144 | 4096 | 0.14 | 272.46 | 18088.12 | 22579.09 | 28.00 | 43.87 | 45.07 | 120183.75 | ⚠️ |
|
||||
| c10_i262144_o4096 | vllm | 10 | 262144 | 4096 | 0.11 | 215.86 | 34100.32 | 38022.94 | 37.18 | 56.07 | 62.00 | 157944.21 | ⚠️ |
|
||||
| c25_i65536_o1024 | sglang | 25 | 65536 | 1024 | 0.67 | 346.70 | 19996.43 | 24478.11 | 149.27 | 395.00 | 2037.09 | 69586.94 | ❌ |
|
||||
| c25_i65536_o1024 | vllm | 25 | 65536 | 1024 | 0.64 | 332.61 | 24861.20 | 28942.75 | 72.36 | 196.24 | 271.32 | 72689.84 | ❌ |
|
||||
| c25_i65536_o256 | sglang | 25 | 65536 | 256 | 0.71 | 84.45 | 27219.06 | 31612.70 | 292.84 | 801.18 | 938.43 | 68854.26 | ❌ |
|
||||
| c25_i65536_o256 | vllm | 25 | 65536 | 256 | 0.81 | 95.60 | 24902.72 | 28974.45 | 180.78 | 273.59 | 278.16 | 60781.45 | ❌ |
|
||||
| c25_i65536_o4096 | sglang | 25 | 65536 | 4096 | 0.41 | 950.01 | 18559.58 | 23249.71 | 23.78 | 42.50 | 82.84 | 96670.12 | ⚠️ |
|
||||
| c25_i65536_o4096 | vllm | 25 | 65536 | 4096 | 0.32 | 741.71 | 24847.97 | 28840.96 | 28.80 | 46.66 | 63.40 | 129654.04 | ⚠️ |
|
||||
| c2_i131072_o1024 | sglang | 2 | 131072 | 1024 | 0.60 | 136.10 | 479.99 | 482.72 | 8.56 | 9.72 | 9.79 | 5480.98 | ✅ |
|
||||
| c2_i131072_o1024 | vllm | 2 | 131072 | 1024 | 0.70 | 158.36 | 676.14 | 695.90 | 6.49 | 8.20 | 8.34 | 4803.24 | ✅ |
|
||||
| c2_i131072_o256 | sglang | 2 | 131072 | 256 | 1.43 | 139.48 | 589.14 | 605.33 | 8.29 | 8.96 | 9.05 | 1750.12 | ✅ |
|
||||
| c2_i131072_o256 | vllm | 2 | 131072 | 256 | 1.51 | 147.12 | 684.63 | 700.28 | 7.08 | 9.27 | 9.45 | 1556.46 | ✅ |
|
||||
| c2_i131072_o4096 | sglang | 2 | 131072 | 4096 | 0.11 | 244.38 | 584.43 | 599.22 | 7.53 | 7.61 | 7.62 | 28360.35 | ✅ |
|
||||
| c2_i131072_o4096 | vllm | 2 | 131072 | 4096 | 0.12 | 262.22 | 671.34 | 688.15 | 7.09 | 7.17 | 7.18 | 26496.04 | ✅ |
|
||||
| c2_i262144_o1024 | sglang | 2 | 262144 | 1024 | 0.51 | 114.94 | 1007.07 | 1024.10 | 8.36 | 9.03 | 9.06 | 6240.65 | ✅ |
|
||||
| c2_i262144_o1024 | vllm | 2 | 262144 | 1024 | 0.19 | 43.14 | 8722.76 | 9009.70 | 20.17 | 52.24 | 57.91 | 13826.13 | ⚠️ |
|
||||
| c2_i262144_o256 | sglang | 2 | 262144 | 256 | 0.29 | 28.44 | 9538.91 | 9980.78 | 14.10 | 26.98 | 29.16 | 11941.34 | ⚠️ |
|
||||
| c2_i262144_o256 | vllm | 2 | 262144 | 256 | 0.23 | 22.74 | 8625.18 | 8952.31 | 10.18 | 23.01 | 25.44 | 10378.60 | ⚠️ |
|
||||
| c2_i262144_o4096 | sglang | 2 | 262144 | 4096 | 0.08 | 189.78 | 9453.57 | 9896.38 | 7.82 | 8.48 | 8.60 | 38723.14 | ⚠️ |
|
||||
| c2_i262144_o4096 | vllm | 2 | 262144 | 4096 | 0.08 | 180.76 | 8759.63 | 9062.80 | 8.63 | 11.53 | 12.01 | 35881.75 | ⚠️ |
|
||||
| c5_i131072_o1024 | sglang | 5 | 131072 | 1024 | 0.87 | 321.54 | 1231.23 | 1298.36 | 9.03 | 10.47 | 10.71 | 7298.25 | ✅ |
|
||||
| c5_i131072_o1024 | vllm | 5 | 131072 | 1024 | 0.90 | 331.65 | 856.18 | 917.80 | 8.93 | 10.48 | 10.62 | 6989.27 | ✅ |
|
||||
| c5_i131072_o256 | sglang | 5 | 131072 | 256 | 1.05 | 146.97 | 3270.23 | 4685.65 | 19.91 | 40.81 | 46.10 | 7815.89 | ⚠️ |
|
||||
| c5_i131072_o256 | vllm | 5 | 131072 | 256 | 0.39 | 54.07 | 9384.23 | 9677.91 | 51.54 | 107.16 | 117.25 | 19860.69 | ❌ |
|
||||
| c5_i131072_o4096 | sglang | 5 | 131072 | 4096 | 0.20 | 378.76 | 4667.56 | 6275.51 | 11.98 | 22.07 | 23.60 | 31176.81 | ⚠️ |
|
||||
| c5_i131072_o4096 | vllm | 5 | 131072 | 4096 | 0.23 | 447.85 | 1054.01 | 1104.66 | 8.48 | 9.13 | 9.14 | 28818.20 | ✅ |
|
||||
| c5_i262144_o1024 | sglang | 5 | 262144 | 1024 | 0.15 | 56.42 | 20522.02 | 25944.29 | 54.11 | 118.98 | 124.02 | 53444.22 | ❌ |
|
||||
| c5_i262144_o1024 | vllm | 5 | 262144 | 1024 | 0.16 | 60.32 | 22564.89 | 26639.92 | 47.49 | 100.69 | 109.86 | 53790.59 | ⚠️ |
|
||||
| c5_i262144_o256 | sglang | 5 | 262144 | 256 | 0.16 | 21.94 | 25875.40 | 30596.95 | 97.05 | 231.75 | 252.98 | 43756.53 | ❌ |
|
||||
| c5_i262144_o256 | vllm | 5 | 262144 | 256 | 0.19 | 25.85 | 21986.99 | 26381.86 | 123.42 | 202.78 | 215.56 | 45761.90 | ❌ |
|
||||
| c5_i262144_o4096 | sglang | 5 | 262144 | 4096 | 0.11 | 209.50 | 16539.50 | 20323.40 | 25.05 | 56.69 | 60.22 | 60383.57 | ⚠️ |
|
||||
| c5_i262144_o4096 | vllm | 5 | 262144 | 4096 | 0.11 | 205.12 | 22171.78 | 26533.69 | 25.69 | 63.32 | 68.33 | 64086.27 | ⚠️ |
|
||||
|
||||
## Notes
|
||||
|
||||
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `docs/SLO_STANDARDS.md`).
|
||||
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
||||
@ -0,0 +1,35 @@
|
||||
# H200 SGLANG Comparison Benchmark Report
|
||||
|
||||
- Result root: `/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/sglang`
|
||||
- Model: `/data/models/DeepSeek-V4-Flash`
|
||||
- Backend: SGLANG (TP=8)
|
||||
- Benchmark client: `sglang.bench_serving --backend sglang`
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| c10_i131072_o1024 | 128k | 10 | 131072 | 1024 | 29.58 | 20 | 0.68 | 49838.58 | 319.74 | 50158.31 | 1968.70 | 4147.51 | 7409.82 | 25.69 | 49.89 | 58.92 | 13268.98 | 25531.40 | 26432.66 | ⚠️ |
|
||||
| c10_i131072_o256 | 128k | 10 | 131072 | 256 | 59.65 | 20 | 0.34 | 24716.54 | 42.70 | 24759.24 | 9130.87 | 26046.62 | 26064.38 | 1374.50 | 1810.56 | 19553.62 | 29314.12 | 58034.97 | 58326.41 | ❌ |
|
||||
| c10_i131072_o4096 | 128k | 10 | 131072 | 4096 | 73.62 | 20 | 0.27 | 20027.92 | 517.96 | 20545.89 | 2624.32 | 6177.42 | 6920.28 | 14.21 | 19.92 | 25.87 | 28500.68 | 53466.72 | 54286.63 | ⚠️ |
|
||||
| c2_i131072_o1024 | 128k | 2 | 131072 | 1024 | 6.62 | 4 | 0.60 | 36602.18 | 136.10 | 36738.28 | 363.59 | 479.99 | 482.72 | 8.56 | 9.72 | 9.79 | 2166.16 | 4983.98 | 5480.98 | ✅ |
|
||||
| c2_i131072_o256 | 128k | 2 | 131072 | 256 | 2.79 | 4 | 1.43 | 86882.00 | 139.48 | 87021.48 | 423.87 | 589.14 | 605.33 | 8.29 | 8.96 | 9.05 | 1238.88 | 1727.94 | 1750.12 | ✅ |
|
||||
| c2_i131072_o4096 | 128k | 2 | 131072 | 4096 | 37.21 | 4 | 0.11 | 6512.27 | 244.38 | 6756.65 | 417.65 | 584.43 | 599.22 | 7.53 | 7.61 | 7.62 | 17481.60 | 27763.29 | 28360.35 | ✅ |
|
||||
| c5_i131072_o1024 | 128k | 5 | 131072 | 1024 | 11.51 | 10 | 0.87 | 64181.45 | 321.54 | 64502.99 | 747.50 | 1231.23 | 1298.36 | 9.03 | 10.47 | 10.71 | 4011.50 | 6687.19 | 7298.25 | ✅ |
|
||||
| c5_i131072_o256 | 128k | 5 | 131072 | 256 | 9.50 | 10 | 1.05 | 77752.11 | 146.97 | 77899.07 | 1222.22 | 3270.23 | 4685.65 | 19.91 | 40.81 | 46.10 | 4366.18 | 7814.85 | 7815.89 | ⚠️ |
|
||||
| c5_i131072_o4096 | 128k | 5 | 131072 | 4096 | 50.32 | 10 | 0.20 | 14676.36 | 378.76 | 15055.12 | 1452.15 | 4667.56 | 6275.51 | 11.98 | 22.07 | 23.60 | 19785.61 | 30849.59 | 31176.81 | ⚠️ |
|
||||
| c10_i262144_o1024 | 256k | 10 | 262144 | 1024 | 123.96 | 20 | 0.16 | 22467.00 | 76.30 | 22543.30 | 12461.07 | 39446.96 | 43965.67 | 108.42 | 202.43 | 202.90 | 60222.02 | 119241.30 | 120180.87 | ❌ |
|
||||
| c10_i262144_o256 | 256k | 10 | 262144 | 256 | 136.21 | 20 | 0.15 | 20447.06 | 18.70 | 20465.76 | 21481.91 | 58074.31 | 62652.59 | 2953.70 | 3625.59 | 41185.32 | 67539.67 | 134322.68 | 134628.33 | ❌ |
|
||||
| c10_i262144_o4096 | 256k | 10 | 262144 | 4096 | 139.95 | 20 | 0.14 | 19900.66 | 272.46 | 20173.12 | 8041.92 | 18088.12 | 22579.09 | 28.00 | 43.87 | 45.07 | 61533.12 | 119183.37 | 120183.75 | ⚠️ |
|
||||
| c2_i262144_o1024 | 256k | 2 | 262144 | 1024 | 7.84 | 4 | 0.51 | 64353.10 | 114.94 | 64468.04 | 817.04 | 1007.07 | 1024.10 | 8.36 | 9.03 | 9.06 | 2623.61 | 5714.71 | 6240.65 | ✅ |
|
||||
| c2_i262144_o256 | 256k | 2 | 262144 | 256 | 13.68 | 4 | 0.29 | 36885.02 | 28.44 | 36913.47 | 4623.18 | 9538.91 | 9980.78 | 14.10 | 26.98 | 29.16 | 6497.61 | 11731.18 | 11941.34 | ⚠️ |
|
||||
| c2_i262144_o4096 | 256k | 2 | 262144 | 4096 | 47.91 | 4 | 0.08 | 10528.36 | 189.78 | 10718.13 | 4475.72 | 9453.57 | 9896.38 | 7.82 | 8.48 | 8.60 | 22625.15 | 38039.50 | 38723.14 | ⚠️ |
|
||||
| c5_i262144_o1024 | 256k | 5 | 262144 | 1024 | 65.58 | 10 | 0.15 | 21254.00 | 56.42 | 21310.42 | 9553.36 | 20522.02 | 25944.29 | 54.11 | 118.98 | 124.02 | 29705.24 | 53300.68 | 53444.22 | ❌ |
|
||||
| c5_i262144_o256 | 256k | 5 | 262144 | 256 | 63.62 | 10 | 0.16 | 21909.37 | 21.94 | 21931.32 | 12937.52 | 25875.40 | 30596.95 | 97.05 | 231.75 | 252.98 | 27766.49 | 43745.44 | 43756.53 | ❌ |
|
||||
| c5_i262144_o4096 | 256k | 5 | 262144 | 4096 | 90.98 | 10 | 0.11 | 15321.51 | 209.50 | 15531.02 | 6888.18 | 16539.50 | 20323.40 | 25.05 | 56.69 | 60.22 | 40104.55 | 58680.97 | 60383.57 | ⚠️ |
|
||||
| c25_i65536_o1024 | 64k | 25 | 65536 | 1024 | 74.65 | 50 | 0.67 | 24668.22 | 346.70 | 25014.92 | 6120.96 | 19996.43 | 24478.11 | 149.27 | 395.00 | 2037.09 | 34945.80 | 68400.96 | 69586.94 | ❌ |
|
||||
| c25_i65536_o256 | 64k | 25 | 65536 | 256 | 70.02 | 50 | 0.71 | 26299.26 | 84.45 | 26383.70 | 9330.86 | 27219.06 | 31612.70 | 292.84 | 801.18 | 938.43 | 34392.33 | 68802.89 | 68854.26 | ❌ |
|
||||
| c25_i65536_o4096 | 64k | 25 | 65536 | 4096 | 122.10 | 50 | 0.41 | 15081.87 | 950.01 | 16031.88 | 5499.83 | 18559.58 | 23249.71 | 23.78 | 42.50 | 82.84 | 52152.37 | 93852.74 | 96670.12 | ⚠️ |
|
||||
|
||||
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,35 @@
|
||||
# H200 VLLM Comparison Benchmark Report
|
||||
|
||||
- Result root: `/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/vllm`
|
||||
- Model: `/data/models/DeepSeek-V4-Flash`
|
||||
- Backend: VLLM (TP=8)
|
||||
- Benchmark client: `sglang.bench_serving --backend vllm`
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| c10_i131072_o1024 | 128k | 10 | 131072 | 1024 | 51.67 | 20 | 0.39 | 28537.46 | 183.08 | 28720.55 | 2870.79 | 6596.15 | 6603.12 | 47.10 | 77.53 | 84.06 | 24302.31 | 47488.34 | 48426.64 | ⚠️ |
|
||||
| c10_i131072_o256 | 128k | 10 | 131072 | 256 | 52.96 | 20 | 0.38 | 27838.09 | 48.09 | 27886.18 | 10289.92 | 23250.03 | 24238.33 | 150.22 | 299.91 | 301.72 | 25961.91 | 51418.73 | 51439.08 | ❌ |
|
||||
| c10_i131072_o4096 | 128k | 10 | 131072 | 4096 | 85.53 | 20 | 0.23 | 17238.00 | 445.81 | 17683.81 | 937.55 | 2096.29 | 2168.81 | 18.57 | 21.47 | 21.61 | 34484.49 | 65903.20 | 67041.13 | ✅ |
|
||||
| c2_i131072_o1024 | 128k | 2 | 131072 | 1024 | 5.69 | 4 | 0.70 | 42587.41 | 158.36 | 42745.76 | 384.25 | 676.14 | 695.90 | 6.49 | 8.20 | 8.34 | 1937.38 | 4390.02 | 4803.24 | ✅ |
|
||||
| c2_i131072_o256 | 128k | 2 | 131072 | 256 | 2.64 | 4 | 1.51 | 91644.87 | 147.12 | 91791.99 | 469.80 | 684.63 | 700.28 | 7.08 | 9.27 | 9.45 | 1208.21 | 1556.10 | 1556.46 | ✅ |
|
||||
| c2_i131072_o4096 | 128k | 2 | 131072 | 4096 | 34.68 | 4 | 0.12 | 6987.68 | 262.22 | 7249.90 | 386.27 | 671.34 | 688.15 | 7.09 | 7.17 | 7.18 | 16427.81 | 25979.54 | 26496.04 | ✅ |
|
||||
| c5_i131072_o1024 | 128k | 5 | 131072 | 1024 | 11.16 | 10 | 0.90 | 66199.25 | 331.65 | 66530.90 | 616.34 | 856.18 | 917.80 | 8.93 | 10.48 | 10.62 | 3905.82 | 6409.85 | 6989.27 | ✅ |
|
||||
| c5_i131072_o256 | 128k | 5 | 131072 | 256 | 25.82 | 10 | 0.39 | 28605.35 | 54.07 | 28659.43 | 4405.54 | 9384.23 | 9677.91 | 51.54 | 107.16 | 117.25 | 12185.89 | 19857.41 | 19860.69 | ❌ |
|
||||
| c5_i131072_o4096 | 128k | 5 | 131072 | 4096 | 42.56 | 10 | 0.23 | 17353.52 | 447.85 | 17801.37 | 615.76 | 1054.01 | 1104.66 | 8.48 | 9.13 | 9.14 | 16241.78 | 28711.60 | 28818.20 | ✅ |
|
||||
| c10_i262144_o1024 | 256k | 10 | 262144 | 1024 | 111.77 | 20 | 0.18 | 24919.08 | 84.63 | 25003.71 | 15133.21 | 34130.22 | 38144.40 | 98.87 | 142.57 | 266.08 | 54087.22 | 107356.89 | 108658.89 | ❌ |
|
||||
| c10_i262144_o256 | 256k | 10 | 262144 | 256 | 111.93 | 20 | 0.18 | 24883.63 | 22.76 | 24906.39 | 28922.71 | 50632.44 | 51883.02 | 229.20 | 339.27 | 344.95 | 55262.71 | 110121.03 | 110216.92 | ❌ |
|
||||
| c10_i262144_o4096 | 256k | 10 | 262144 | 4096 | 176.65 | 20 | 0.11 | 15766.52 | 215.86 | 15982.38 | 11662.77 | 34100.32 | 38022.94 | 37.18 | 56.07 | 62.00 | 79905.41 | 156158.99 | 157944.21 | ⚠️ |
|
||||
| c2_i262144_o1024 | 256k | 2 | 262144 | 1024 | 20.88 | 4 | 0.19 | 24156.04 | 43.14 | 24199.18 | 5483.04 | 8722.76 | 9009.70 | 20.17 | 52.24 | 57.91 | 9282.52 | 13247.86 | 13826.13 | ⚠️ |
|
||||
| c2_i262144_o256 | 256k | 2 | 262144 | 256 | 17.11 | 4 | 0.23 | 29488.06 | 22.74 | 29510.80 | 6791.08 | 8625.18 | 8952.31 | 10.18 | 23.01 | 25.44 | 8281.06 | 10221.13 | 10378.60 | ⚠️ |
|
||||
| c2_i262144_o4096 | 256k | 2 | 262144 | 4096 | 50.30 | 4 | 0.08 | 10027.95 | 180.76 | 10208.71 | 5468.77 | 8759.63 | 9062.80 | 8.63 | 11.53 | 12.01 | 23992.03 | 35306.58 | 35881.75 | ⚠️ |
|
||||
| c5_i262144_o1024 | 256k | 5 | 262144 | 1024 | 61.34 | 10 | 0.16 | 22724.99 | 60.32 | 22785.31 | 11196.96 | 22564.89 | 26639.92 | 47.49 | 100.69 | 109.86 | 28871.28 | 47913.60 | 53790.59 | ⚠️ |
|
||||
| c5_i262144_o256 | 256k | 5 | 262144 | 256 | 54.01 | 10 | 0.19 | 25809.33 | 25.85 | 25835.18 | 9094.42 | 21986.99 | 26381.86 | 123.42 | 202.78 | 215.56 | 26745.69 | 45719.33 | 45761.90 | ❌ |
|
||||
| c5_i262144_o4096 | 256k | 5 | 262144 | 4096 | 92.92 | 10 | 0.11 | 15001.24 | 205.12 | 15206.37 | 9122.40 | 22171.78 | 26533.69 | 25.69 | 63.32 | 68.33 | 41465.64 | 61304.00 | 64086.27 | ⚠️ |
|
||||
| c25_i65536_o1024 | 64k | 25 | 65536 | 1024 | 77.81 | 50 | 0.64 | 23665.15 | 332.61 | 23997.76 | 8358.86 | 24861.20 | 28942.75 | 72.36 | 196.24 | 271.32 | 36604.32 | 71227.08 | 72689.84 | ❌ |
|
||||
| c25_i65536_o256 | 64k | 25 | 65536 | 256 | 61.85 | 50 | 0.81 | 29772.41 | 95.60 | 29868.01 | 11442.71 | 24902.72 | 28974.45 | 180.78 | 273.59 | 278.16 | 30201.94 | 60181.25 | 60781.45 | ❌ |
|
||||
| c25_i65536_o4096 | 64k | 25 | 65536 | 4096 | 156.39 | 50 | 0.32 | 11775.02 | 741.71 | 12516.73 | 7847.53 | 24847.97 | 28840.96 | 28.80 | 46.66 | 63.40 | 69020.59 | 126999.05 | 129654.04 | ⚠️ |
|
||||
|
||||
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -93,7 +93,7 @@ run_warmup() {
|
||||
fi
|
||||
|
||||
log "warming up ${backend} (input=${input_len}, output=${output_len}, num=1)"
|
||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \
|
||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \
|
||||
--backend "$backend" \
|
||||
--port "$port" \
|
||||
--input-len "$input_len" \
|
||||
@ -201,7 +201,7 @@ parse_backend() {
|
||||
local backend="$1"
|
||||
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
||||
log "parsing ${backend} results in ${result_root}"
|
||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \
|
||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \
|
||||
>> "${result_root}/logs/parse.log" 2>&1 || {
|
||||
log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log"
|
||||
}
|
||||
@ -282,7 +282,7 @@ parse_backend sglang
|
||||
parse_backend vllm
|
||||
|
||||
log "generating comparison report"
|
||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \
|
||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \
|
||||
--sglang "${RESULT_BASE}/${RUN_ID}/sglang" \
|
||||
--vllm "${RESULT_BASE}/${RUN_ID}/vllm" \
|
||||
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
||||
|
||||
@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send a small number of warmup requests to a running backend.
|
||||
|
||||
Uses sglang.bench_serving with a single request so that the same code path
|
||||
(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real
|
||||
benchmark begins. Discards the output.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None:
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp:
|
||||
cmd = [
|
||||
str(env_python),
|
||||
"-m",
|
||||
"sglang.bench_serving",
|
||||
"--backend",
|
||||
backend,
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--dataset-name",
|
||||
"random",
|
||||
"--random-input-len",
|
||||
str(input_len),
|
||||
"--random-output-len",
|
||||
str(output_len),
|
||||
"--num-prompts",
|
||||
str(num),
|
||||
"--max-concurrency",
|
||||
"1",
|
||||
"--request-rate",
|
||||
"10000",
|
||||
"--output-file",
|
||||
tmp.name,
|
||||
"--output-details",
|
||||
]
|
||||
print(f"[warmup] {' '.join(cmd)}", flush=True)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print("[warmup] FAILED", file=sys.stderr)
|
||||
print(result.stdout, file=sys.stderr)
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Warmup a serving backend.")
|
||||
parser.add_argument("--backend", required=True, choices=["sglang", "vllm"])
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--input-len", type=int, required=True)
|
||||
parser.add_argument("--output-len", type=int, required=True)
|
||||
parser.add_argument("--num", type=int, default=1)
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_warmup(
|
||||
backend=args.backend,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
input_len=args.input_len,
|
||||
output_len=args.output_len,
|
||||
num=args.num,
|
||||
env_python=Path(args.env_python),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user