diff --git a/BENCHMARK_WORKFLOW.md b/BENCHMARK_WORKFLOW.md index 0553597..4958067 100644 --- a/BENCHMARK_WORKFLOW.md +++ b/BENCHMARK_WORKFLOW.md @@ -192,6 +192,11 @@ The JSON file inside each `bench_results//` directory should follow a stabl "tpot_ms": { "mean": 18.24, "p50": 16.0, "p90": 28.0, "p95": 30.39, "p99": 39.57 }, "itl_ms": { "mean": 70.29, "p50": 60.0, "p90": 110.0, "p95": 130.0, "p99": 160.0 } }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, "raw_requests": [ { "request_id": "uuid-or-index", @@ -214,6 +219,10 @@ The JSON file inside each `bench_results//` directory should follow a stabl - `raw_requests` is optional but recommended when the JSON size is manageable. If a single run produces millions of requests, store per-request data as `raw_outputs/*.jsonl` and keep only aggregated percentiles in `results.json`. - Always include **P50 / P90 / P95 / P99** for TTFT, TPOT, E2E, and ITL. P95 is the primary SLO metric. +- Include an `slo_status` object per scenario indicating whether the scenario meets the relevant SLO (e.g. S2 tier: TTFT P95 < 3000ms, TPOT mean < 50ms). Example: + ```json + "slo_status": { "ttft_p95_ok": true, "tpot_mean_ok": true, "overall": "✅" } + ``` - Keep field names snake_case and consistent across experiments. - Record hardware and engine information explicitly: - `accelerator` / `chip`: the accelerator family, e.g. `NVIDIA H200`, `Kunlun XPU`, `AMD MI300X`. @@ -333,6 +342,7 @@ See `docs/H200_QUICKSTART.md` for a concrete H200 porting walkthrough. - [ ] `/results.json` exists and follows the [Final JSON Schema](#final-json-schema). - [ ] `/results.json` metadata records the `chip`/`accelerator` and `engine`/`backend` used. - [ ] `/results.json` `config` records the exact server launch command/args (`server_args` or `phaseN_server_args`). +- [ ] `/results.json` each scenario records `slo_status` (overall pass/fail/partial against the relevant SLO). - [ ] Raw output filenames include the chip/accelerator and engine when cross-platform runs may collide. - [ ] `/README.md` exists and documents provenance (or the report itself covers provenance). - [ ] Scripts either live under `experiments//` or in `scripts/` (or `scripts//`). diff --git a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py index 721394e..8ae2322 100755 --- a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py +++ b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py @@ -78,6 +78,23 @@ def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: return f"c{concurrency}_i{input_len}_o{output_len}" +def slo_status(metrics: dict) -> dict: + """Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms.""" + ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0 + tpot_ok = metrics["tpot_ms"]["mean"] < 50.0 + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + def append_scenario(results_json: Path, scenario: dict) -> None: with open(results_json, "r", encoding="utf-8") as f: data = json.load(f) @@ -96,12 +113,13 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") f.write("## Results\n\n") - f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") for s in scenarios: cfg = s["config"] m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") f.write( f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " @@ -109,9 +127,10 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N f"{m['total_token_throughput']:.2f} | " f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n" + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" ) f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") def main() -> None: @@ -178,6 +197,7 @@ def main() -> None: "num_prompts": metrics["success"] + metrics["failed"], }, "metrics": metrics, + "slo_status": slo_status(metrics), "raw_file": str(jsonl_path), } scenarios.append(scenario) diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py b/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py index cd1f2f1..b227874 100755 --- a/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py @@ -76,6 +76,23 @@ def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: return f"c{concurrency}_i{input_len}_o{output_len}" +def slo_status(metrics: dict) -> dict: + """Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms.""" + ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0 + tpot_ok = metrics["tpot_ms"]["mean"] < 50.0 + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + def append_scenario(results_json: Path, scenario: dict) -> None: with open(results_json, "r", encoding="utf-8") as f: data = json.load(f) @@ -94,12 +111,13 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N f.write(f"- Benchmark client: `sglang.bench_serving --backend vllm`\n\n") f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") for s in scenarios: cfg = s["config"] m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") f.write( f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " @@ -107,9 +125,10 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N f"{m['total_token_throughput']:.2f} | " f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n" + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" ) f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") def main() -> None: @@ -162,6 +181,7 @@ def main() -> None: "num_prompts": metrics["success"] + metrics["failed"], }, "metrics": metrics, + "slo_status": slo_status(metrics), "raw_file": str(jsonl_path), } scenarios.append(scenario)