diff --git a/experiments/dsv4_h200_256k_4k_probe/compare.py b/experiments/dsv4_h200_256k_4k_probe/compare.py deleted file mode 100755 index 0f4eb51..0000000 --- a/experiments/dsv4_h200_256k_4k_probe/compare.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -"""Generate a side-by-side comparison of SGLang and vLLM results. - -Usage: - python3 compare.py --sglang --vllm \ - [--output comparison.md] -""" -import argparse -import json -from collections import defaultdict -from pathlib import Path - - -def load_result(result_root: Path) -> dict: - path = result_root / "results.json" - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str: - # S2 tier targets from scripts/SLO_STANDARDS.md. - ttft_ok = ttft_p95_ms < 3000.0 - tpot_ok = tpot_mean_ms < 50.0 - if ttft_ok and tpot_ok: - return "✅" - if ttft_ok or tpot_ok: - return "⚠️" - return "❌" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--sglang", type=Path, required=True) - parser.add_argument("--vllm", type=Path, required=True) - parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) - args = parser.parse_args() - - sglang_data = load_result(args.sglang) - vllm_data = load_result(args.vllm) - - by_scenario = defaultdict(dict) - for data in (sglang_data, vllm_data): - backend = data["metadata"]["engine"] - for s in data.get("scenarios", []): - key = s["name"] - by_scenario[key][backend] = s - - with open(args.output, "w", encoding="utf-8") as f: - f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n") - f.write("## Summary\n\n") - f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write("- Hardware: 8x NVIDIA H200 143GB\n") - f.write("- Tensor Parallelism: 8\n") - f.write("- Benchmark client: `sglang.bench_serving`\n") - f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n") - - f.write("## Side-by-side results\n\n") - f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") - f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for scenario_name in sorted(by_scenario.keys()): - for backend in ("sglang", "vllm"): - s = by_scenario[scenario_name].get(backend) - if s is None: - continue - cfg = s["config"] - m = s["metrics"] - status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"]) - f.write( - f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['p99']:.2f} | {status} |\n" - ) - - f.write("\n## Notes\n\n") - f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n") - f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") - - print(f"Wrote comparison to {args.output}") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_256k_4k_probe/parse_backend.py b/experiments/dsv4_h200_256k_4k_probe/parse_backend.py deleted file mode 100755 index 935d870..0000000 --- a/experiments/dsv4_h200_256k_4k_probe/parse_backend.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -"""Parse raw sglang.bench_serving JSONL outputs for one backend. - -Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl -and generates results.json + report.md in the given result root. - -Usage: - python3 parse_backend.py [--backend sglang|vllm] -""" -import argparse -import json -import re -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> dict | None: - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - return json.loads(line) - except json.JSONDecodeError: - continue - return None - - -def compute_metrics(data: dict) -> dict: - completed = data.get("completed", 0) - total = len(data.get("input_lens", [])) - failed = total - completed if total > 0 else 0 - duration_s = data.get("duration", 0.0) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": data.get("request_throughput", 0.0), - "input_token_throughput": data.get("input_throughput", 0.0), - "output_token_throughput": data.get("output_throughput", 0.0), - "total_token_throughput": data.get("total_throughput", 0.0), - "total_input_tokens": data.get("total_input_tokens", 0), - "total_output_tokens": data.get("total_output_tokens", 0), - "e2e_ms": { - "mean": data.get("mean_e2e_latency_ms", 0.0), - "p50": data.get("median_e2e_latency_ms", 0.0), - "p90": data.get("p90_e2e_latency_ms", 0.0), - "p95": data.get("p95_e2e_latency_ms", 0.0), - "p99": data.get("p99_e2e_latency_ms", 0.0), - }, - "ttft_ms": { - "mean": data.get("mean_ttft_ms", 0.0), - "p50": data.get("median_ttft_ms", 0.0), - "p90": data.get("p90_ttft_ms", 0.0), - "p95": data.get("p95_ttft_ms", 0.0), - "p99": data.get("p99_ttft_ms", 0.0), - }, - "tpot_ms": { - "mean": data.get("mean_tpot_ms", 0.0), - "p50": data.get("median_tpot_ms", 0.0), - "p90": data.get("p90_tpot_ms", 0.0), - "p95": data.get("p95_tpot_ms", 0.0), - "p99": data.get("p99_tpot_ms", 0.0), - }, - "itl_ms": { - "mean": data.get("mean_itl_ms", 0.0), - "p50": data.get("median_itl_ms", 0.0), - "p90": data.get("p90_itl_ms", 0.0), - "p95": data.get("p95_itl_ms", 0.0), - "p99": data.get("p99_itl_ms", 0.0), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict) -> dict: - """Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms.""" - ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0 - tpot_ok = metrics["tpot_ms"]["mean"] < 50.0 - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n") - f.write(f"- Result root: `{result_root}`\n") - f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write(f"- Backend: {backend.upper()} (TP=8)\n") - f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("result_root", type=Path) - parser.add_argument("--backend", default=None, choices=["sglang", "vllm"]) - args = parser.parse_args() - - result_root = args.result_root - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - backend = args.backend - if backend is None: - # Infer from filenames if not provided. - for p in raw_dir.iterdir(): - if p.name.startswith("sglang_"): - backend = "sglang" - break - if p.name.startswith("vllm_"): - backend = "vllm" - break - if backend is None: - raise SystemExit("Could not infer backend from raw outputs") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - - label = parts[1] - if label == "sharegpt": - phase = "sharegpt" - dataset = "sharegpt" - else: - phase = label - dataset = "random" - - # The last three numeric tokens are: concurrency, input_len/output_ctx, output_len. - try: - concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1]) - except ValueError: - continue - - data = parse_jsonl(jsonl_path) - if data is None: - continue - - metrics = compute_metrics(data) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "phase": phase, - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": dataset, - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"] = scenarios - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - generate_report(result_root, backend, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_256k_4k_probe/warmup.py b/experiments/dsv4_h200_256k_4k_probe/warmup.py deleted file mode 100755 index 75dbe5f..0000000 --- a/experiments/dsv4_h200_256k_4k_probe/warmup.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -"""Send a small number of warmup requests to a running backend. - -Uses sglang.bench_serving with a single request so that the same code path -(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real -benchmark begins. Discards the output. -""" -import argparse -import subprocess -import sys -import tempfile -from pathlib import Path - - -def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None: - with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp: - cmd = [ - str(env_python), - "-m", - "sglang.bench_serving", - "--backend", - backend, - "--host", - host, - "--port", - str(port), - "--dataset-name", - "random", - "--random-input-len", - str(input_len), - "--random-output-len", - str(output_len), - "--num-prompts", - str(num), - "--max-concurrency", - "1", - "--request-rate", - "10000", - "--output-file", - tmp.name, - "--output-details", - ] - print(f"[warmup] {' '.join(cmd)}", flush=True) - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print("[warmup] FAILED", file=sys.stderr) - print(result.stdout, file=sys.stderr) - print(result.stderr, file=sys.stderr) - sys.exit(1) - print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}") - - -def main(): - parser = argparse.ArgumentParser(description="Warmup a serving backend.") - parser.add_argument("--backend", required=True, choices=["sglang", "vllm"]) - parser.add_argument("--port", type=int, required=True) - parser.add_argument("--input-len", type=int, required=True) - parser.add_argument("--output-len", type=int, required=True) - parser.add_argument("--num", type=int, default=1) - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python") - args = parser.parse_args() - - run_warmup( - backend=args.backend, - host=args.host, - port=args.port, - input_len=args.input_len, - output_len=args.output_len, - num=args.num, - env_python=Path(args.env_python), - ) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_64k_sglang_vs_vllm/compare.py b/experiments/dsv4_h200_64k_sglang_vs_vllm/compare.py deleted file mode 100755 index 0f4eb51..0000000 --- a/experiments/dsv4_h200_64k_sglang_vs_vllm/compare.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -"""Generate a side-by-side comparison of SGLang and vLLM results. - -Usage: - python3 compare.py --sglang --vllm \ - [--output comparison.md] -""" -import argparse -import json -from collections import defaultdict -from pathlib import Path - - -def load_result(result_root: Path) -> dict: - path = result_root / "results.json" - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str: - # S2 tier targets from scripts/SLO_STANDARDS.md. - ttft_ok = ttft_p95_ms < 3000.0 - tpot_ok = tpot_mean_ms < 50.0 - if ttft_ok and tpot_ok: - return "✅" - if ttft_ok or tpot_ok: - return "⚠️" - return "❌" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--sglang", type=Path, required=True) - parser.add_argument("--vllm", type=Path, required=True) - parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) - args = parser.parse_args() - - sglang_data = load_result(args.sglang) - vllm_data = load_result(args.vllm) - - by_scenario = defaultdict(dict) - for data in (sglang_data, vllm_data): - backend = data["metadata"]["engine"] - for s in data.get("scenarios", []): - key = s["name"] - by_scenario[key][backend] = s - - with open(args.output, "w", encoding="utf-8") as f: - f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n") - f.write("## Summary\n\n") - f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write("- Hardware: 8x NVIDIA H200 143GB\n") - f.write("- Tensor Parallelism: 8\n") - f.write("- Benchmark client: `sglang.bench_serving`\n") - f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n") - - f.write("## Side-by-side results\n\n") - f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") - f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for scenario_name in sorted(by_scenario.keys()): - for backend in ("sglang", "vllm"): - s = by_scenario[scenario_name].get(backend) - if s is None: - continue - cfg = s["config"] - m = s["metrics"] - status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"]) - f.write( - f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['p99']:.2f} | {status} |\n" - ) - - f.write("\n## Notes\n\n") - f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n") - f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") - - print(f"Wrote comparison to {args.output}") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_64k_sglang_vs_vllm/parse_backend.py b/experiments/dsv4_h200_64k_sglang_vs_vllm/parse_backend.py deleted file mode 100755 index 935d870..0000000 --- a/experiments/dsv4_h200_64k_sglang_vs_vllm/parse_backend.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -"""Parse raw sglang.bench_serving JSONL outputs for one backend. - -Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl -and generates results.json + report.md in the given result root. - -Usage: - python3 parse_backend.py [--backend sglang|vllm] -""" -import argparse -import json -import re -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> dict | None: - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - return json.loads(line) - except json.JSONDecodeError: - continue - return None - - -def compute_metrics(data: dict) -> dict: - completed = data.get("completed", 0) - total = len(data.get("input_lens", [])) - failed = total - completed if total > 0 else 0 - duration_s = data.get("duration", 0.0) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": data.get("request_throughput", 0.0), - "input_token_throughput": data.get("input_throughput", 0.0), - "output_token_throughput": data.get("output_throughput", 0.0), - "total_token_throughput": data.get("total_throughput", 0.0), - "total_input_tokens": data.get("total_input_tokens", 0), - "total_output_tokens": data.get("total_output_tokens", 0), - "e2e_ms": { - "mean": data.get("mean_e2e_latency_ms", 0.0), - "p50": data.get("median_e2e_latency_ms", 0.0), - "p90": data.get("p90_e2e_latency_ms", 0.0), - "p95": data.get("p95_e2e_latency_ms", 0.0), - "p99": data.get("p99_e2e_latency_ms", 0.0), - }, - "ttft_ms": { - "mean": data.get("mean_ttft_ms", 0.0), - "p50": data.get("median_ttft_ms", 0.0), - "p90": data.get("p90_ttft_ms", 0.0), - "p95": data.get("p95_ttft_ms", 0.0), - "p99": data.get("p99_ttft_ms", 0.0), - }, - "tpot_ms": { - "mean": data.get("mean_tpot_ms", 0.0), - "p50": data.get("median_tpot_ms", 0.0), - "p90": data.get("p90_tpot_ms", 0.0), - "p95": data.get("p95_tpot_ms", 0.0), - "p99": data.get("p99_tpot_ms", 0.0), - }, - "itl_ms": { - "mean": data.get("mean_itl_ms", 0.0), - "p50": data.get("median_itl_ms", 0.0), - "p90": data.get("p90_itl_ms", 0.0), - "p95": data.get("p95_itl_ms", 0.0), - "p99": data.get("p99_itl_ms", 0.0), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict) -> dict: - """Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms.""" - ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0 - tpot_ok = metrics["tpot_ms"]["mean"] < 50.0 - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n") - f.write(f"- Result root: `{result_root}`\n") - f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write(f"- Backend: {backend.upper()} (TP=8)\n") - f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("result_root", type=Path) - parser.add_argument("--backend", default=None, choices=["sglang", "vllm"]) - args = parser.parse_args() - - result_root = args.result_root - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - backend = args.backend - if backend is None: - # Infer from filenames if not provided. - for p in raw_dir.iterdir(): - if p.name.startswith("sglang_"): - backend = "sglang" - break - if p.name.startswith("vllm_"): - backend = "vllm" - break - if backend is None: - raise SystemExit("Could not infer backend from raw outputs") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - - label = parts[1] - if label == "sharegpt": - phase = "sharegpt" - dataset = "sharegpt" - else: - phase = label - dataset = "random" - - # The last three numeric tokens are: concurrency, input_len/output_ctx, output_len. - try: - concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1]) - except ValueError: - continue - - data = parse_jsonl(jsonl_path) - if data is None: - continue - - metrics = compute_metrics(data) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "phase": phase, - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": dataset, - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"] = scenarios - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - generate_report(result_root, backend, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_64k_sglang_vs_vllm/warmup.py b/experiments/dsv4_h200_64k_sglang_vs_vllm/warmup.py deleted file mode 100755 index 75dbe5f..0000000 --- a/experiments/dsv4_h200_64k_sglang_vs_vllm/warmup.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -"""Send a small number of warmup requests to a running backend. - -Uses sglang.bench_serving with a single request so that the same code path -(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real -benchmark begins. Discards the output. -""" -import argparse -import subprocess -import sys -import tempfile -from pathlib import Path - - -def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None: - with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp: - cmd = [ - str(env_python), - "-m", - "sglang.bench_serving", - "--backend", - backend, - "--host", - host, - "--port", - str(port), - "--dataset-name", - "random", - "--random-input-len", - str(input_len), - "--random-output-len", - str(output_len), - "--num-prompts", - str(num), - "--max-concurrency", - "1", - "--request-rate", - "10000", - "--output-file", - tmp.name, - "--output-details", - ] - print(f"[warmup] {' '.join(cmd)}", flush=True) - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print("[warmup] FAILED", file=sys.stderr) - print(result.stdout, file=sys.stderr) - print(result.stderr, file=sys.stderr) - sys.exit(1) - print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}") - - -def main(): - parser = argparse.ArgumentParser(description="Warmup a serving backend.") - parser.add_argument("--backend", required=True, choices=["sglang", "vllm"]) - parser.add_argument("--port", type=int, required=True) - parser.add_argument("--input-len", type=int, required=True) - parser.add_argument("--output-len", type=int, required=True) - parser.add_argument("--num", type=int, default=1) - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python") - args = parser.parse_args() - - run_warmup( - backend=args.backend, - host=args.host, - port=args.port, - input_len=args.input_len, - output_len=args.output_len, - num=args.num, - env_python=Path(args.env_python), - ) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_sglang_vs_vllm/compare.py b/experiments/dsv4_h200_sglang_vs_vllm/compare.py deleted file mode 100755 index 0f4eb51..0000000 --- a/experiments/dsv4_h200_sglang_vs_vllm/compare.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -"""Generate a side-by-side comparison of SGLang and vLLM results. - -Usage: - python3 compare.py --sglang --vllm \ - [--output comparison.md] -""" -import argparse -import json -from collections import defaultdict -from pathlib import Path - - -def load_result(result_root: Path) -> dict: - path = result_root / "results.json" - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str: - # S2 tier targets from scripts/SLO_STANDARDS.md. - ttft_ok = ttft_p95_ms < 3000.0 - tpot_ok = tpot_mean_ms < 50.0 - if ttft_ok and tpot_ok: - return "✅" - if ttft_ok or tpot_ok: - return "⚠️" - return "❌" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--sglang", type=Path, required=True) - parser.add_argument("--vllm", type=Path, required=True) - parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) - args = parser.parse_args() - - sglang_data = load_result(args.sglang) - vllm_data = load_result(args.vllm) - - by_scenario = defaultdict(dict) - for data in (sglang_data, vllm_data): - backend = data["metadata"]["engine"] - for s in data.get("scenarios", []): - key = s["name"] - by_scenario[key][backend] = s - - with open(args.output, "w", encoding="utf-8") as f: - f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n") - f.write("## Summary\n\n") - f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write("- Hardware: 8x NVIDIA H200 143GB\n") - f.write("- Tensor Parallelism: 8\n") - f.write("- Benchmark client: `sglang.bench_serving`\n") - f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n") - - f.write("## Side-by-side results\n\n") - f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") - f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for scenario_name in sorted(by_scenario.keys()): - for backend in ("sglang", "vllm"): - s = by_scenario[scenario_name].get(backend) - if s is None: - continue - cfg = s["config"] - m = s["metrics"] - status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"]) - f.write( - f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['p99']:.2f} | {status} |\n" - ) - - f.write("\n## Notes\n\n") - f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n") - f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") - - print(f"Wrote comparison to {args.output}") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py deleted file mode 100755 index d0c195f..0000000 --- a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 -"""Parse raw sglang.bench_serving JSONL outputs for one backend. - -Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl -and generates results.json + report.md in the given result root. - -Usage: - python3 parse_backend.py [--backend sglang|vllm] -""" -import argparse -import json -import re -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> dict | None: - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - return json.loads(line) - except json.JSONDecodeError: - continue - return None - - -def compute_metrics(data: dict) -> dict: - completed = data.get("completed", 0) - total = len(data.get("input_lens", [])) - failed = total - completed if total > 0 else 0 - duration_s = data.get("duration", 0.0) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": data.get("request_throughput", 0.0), - "input_token_throughput": data.get("input_throughput", 0.0), - "output_token_throughput": data.get("output_throughput", 0.0), - "total_token_throughput": data.get("total_throughput", 0.0), - "total_input_tokens": data.get("total_input_tokens", 0), - "total_output_tokens": data.get("total_output_tokens", 0), - "e2e_ms": { - "mean": data.get("mean_e2e_latency_ms", 0.0), - "p50": data.get("median_e2e_latency_ms", 0.0), - "p90": data.get("p90_e2e_latency_ms", 0.0), - "p95": data.get("p95_e2e_latency_ms", 0.0), - "p99": data.get("p99_e2e_latency_ms", 0.0), - }, - "ttft_ms": { - "mean": data.get("mean_ttft_ms", 0.0), - "p50": data.get("median_ttft_ms", 0.0), - "p90": data.get("p90_ttft_ms", 0.0), - "p95": data.get("p95_ttft_ms", 0.0), - "p99": data.get("p99_ttft_ms", 0.0), - }, - "tpot_ms": { - "mean": data.get("mean_tpot_ms", 0.0), - "p50": data.get("median_tpot_ms", 0.0), - "p90": data.get("p90_tpot_ms", 0.0), - "p95": data.get("p95_tpot_ms", 0.0), - "p99": data.get("p99_tpot_ms", 0.0), - }, - "itl_ms": { - "mean": data.get("mean_itl_ms", 0.0), - "p50": data.get("median_itl_ms", 0.0), - "p90": data.get("p90_itl_ms", 0.0), - "p95": data.get("p95_itl_ms", 0.0), - "p99": data.get("p99_itl_ms", 0.0), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict) -> dict: - """Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms.""" - ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0 - tpot_ok = metrics["tpot_ms"]["mean"] < 50.0 - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n") - f.write(f"- Result root: `{result_root}`\n") - f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n") - f.write(f"- Backend: {backend.upper()} (TP=8)\n") - f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("result_root", type=Path) - parser.add_argument("--backend", default=None, choices=["sglang", "vllm"]) - args = parser.parse_args() - - result_root = args.result_root - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - backend = args.backend - if backend is None: - # Infer from filenames if not provided. - for p in raw_dir.iterdir(): - if p.name.startswith("sglang_"): - backend = "sglang" - break - if p.name.startswith("vllm_"): - backend = "vllm" - break - if backend is None: - raise SystemExit("Could not infer backend from raw outputs") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - - label = parts[1] - if label.startswith("phase"): - phase = label - dataset = "random" - elif label == "sharegpt": - phase = "sharegpt" - dataset = "sharegpt" - else: - continue - - # The last three numeric tokens are: concurrency, input_len/output_ctx, output_len. - try: - concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1]) - except ValueError: - continue - - data = parse_jsonl(jsonl_path) - if data is None: - continue - - metrics = compute_metrics(data) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "phase": phase, - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": dataset, - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"] = scenarios - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - generate_report(result_root, backend, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_sglang_vs_vllm/warmup.py b/experiments/dsv4_h200_sglang_vs_vllm/warmup.py deleted file mode 100755 index 75dbe5f..0000000 --- a/experiments/dsv4_h200_sglang_vs_vllm/warmup.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -"""Send a small number of warmup requests to a running backend. - -Uses sglang.bench_serving with a single request so that the same code path -(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real -benchmark begins. Discards the output. -""" -import argparse -import subprocess -import sys -import tempfile -from pathlib import Path - - -def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None: - with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp: - cmd = [ - str(env_python), - "-m", - "sglang.bench_serving", - "--backend", - backend, - "--host", - host, - "--port", - str(port), - "--dataset-name", - "random", - "--random-input-len", - str(input_len), - "--random-output-len", - str(output_len), - "--num-prompts", - str(num), - "--max-concurrency", - "1", - "--request-rate", - "10000", - "--output-file", - tmp.name, - "--output-details", - ] - print(f"[warmup] {' '.join(cmd)}", flush=True) - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print("[warmup] FAILED", file=sys.stderr) - print(result.stdout, file=sys.stderr) - print(result.stderr, file=sys.stderr) - sys.exit(1) - print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}") - - -def main(): - parser = argparse.ArgumentParser(description="Warmup a serving backend.") - parser.add_argument("--backend", required=True, choices=["sglang", "vllm"]) - parser.add_argument("--port", type=int, required=True) - parser.add_argument("--input-len", type=int, required=True) - parser.add_argument("--output-len", type=int, required=True) - parser.add_argument("--num", type=int, default=1) - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python") - args = parser.parse_args() - - run_warmup( - backend=args.backend, - host=args.host, - port=args.port, - input_len=args.input_len, - output_len=args.output_len, - num=args.num, - env_python=Path(args.env_python), - ) - - -if __name__ == "__main__": - main()