7e614fb702
- Add start_dspark.sh with --spec-method dspark --spec-tokens 5 - Add BACKENDS env var to run only selected backends - Fix client/backend mapping for DSpark (bench client uses vllm, parser uses vllm) - Add three-way compare.py for sglang/vllm/dspark - Reuse existing sglang/vllm results from 20260708-100016 - Add DSpark long-context results and comparison
101 lines
4.1 KiB
Python
Executable File
101 lines
4.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate a side-by-side comparison of SGLang, vLLM, and DSpark results.
|
|
|
|
Usage:
|
|
python3 compare.py --sglang <sglang_root> --vllm <vllm_root> --dspark <dspark_root> \
|
|
[--output comparison.md]
|
|
"""
|
|
import argparse
|
|
import json
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
def load_result(result_root: Path) -> dict:
|
|
path = result_root / "results.json"
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str:
|
|
ttft_ok = ttft_p95_ms < ttft_limit_ms
|
|
tpot_ok = tpot_mean_ms < tpot_limit_ms
|
|
if ttft_ok and tpot_ok:
|
|
return "✅"
|
|
if ttft_ok or tpot_ok:
|
|
return "⚠️"
|
|
return "❌"
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--sglang", type=Path, required=True)
|
|
parser.add_argument("--vllm", type=Path, required=True)
|
|
parser.add_argument("--dspark", type=Path, required=True)
|
|
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
|
parser.add_argument("--ttft-limit", type=float, default=3000.0)
|
|
parser.add_argument("--tpot-limit", type=float, default=50.0)
|
|
args = parser.parse_args()
|
|
|
|
sglang_data = load_result(args.sglang)
|
|
vllm_data = load_result(args.vllm)
|
|
dspark_data = load_result(args.dspark)
|
|
|
|
model = sglang_data.get("metadata", {}).get("model", "unknown")
|
|
hardware = sglang_data.get("metadata", {}).get("hardware", "unknown")
|
|
|
|
# Map engine names to display labels.
|
|
engine_to_label = {
|
|
"sglang": "sglang",
|
|
"vllm": "vllm",
|
|
"vllm-dspark": "dspark",
|
|
}
|
|
|
|
by_scenario = defaultdict(dict)
|
|
for data in (sglang_data, vllm_data, dspark_data):
|
|
engine = data["metadata"]["engine"]
|
|
label = engine_to_label.get(engine, engine)
|
|
for s in data.get("scenarios", []):
|
|
key = s["name"]
|
|
by_scenario[key][label] = s
|
|
|
|
with open(args.output, "w", encoding="utf-8") as f:
|
|
f.write(f"# SGLang vs vLLM vs DSpark long-context matrix ({hardware})\n\n")
|
|
f.write("## Summary\n\n")
|
|
f.write(f"- Model: `{model}`\n")
|
|
f.write(f"- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark`\n")
|
|
f.write(f"- Hardware: {hardware}\n")
|
|
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
|
f.write("- DSpark flags: `--spec-method dspark --spec-model <model> --spec-tokens 5`\n")
|
|
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
|
|
|
|
f.write("## Side-by-side results\n\n")
|
|
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
|
|
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
|
|
|
for scenario_name in sorted(by_scenario.keys()):
|
|
for backend in ("sglang", "vllm", "dspark"):
|
|
s = by_scenario[scenario_name].get(backend)
|
|
if s is None:
|
|
continue
|
|
cfg = s["config"]
|
|
m = s["metrics"]
|
|
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
|
f.write(
|
|
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
|
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
|
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
|
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
|
f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
|
|
)
|
|
|
|
f.write("\n## Notes\n\n")
|
|
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
|
|
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
|
|
|
print(f"Wrote comparison to {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|