feat: add SLO status field to per-backend reports and results.json
- parse_backend.py for sglang_vs_vllm and dspark_vs_default: - compute slo_status per scenario (TTFT P95 < 3000ms, TPOT mean < 50ms) - append slo_status to results.json - add SLO column to report.md - BENCHMARK_WORKFLOW.md documents slo_status in schema and checklist
This commit is contained in:
yy-fighting
parent
b5b636236d
commit
0bb8adf2d9
@ -192,6 +192,11 @@ The JSON file inside each `bench_results/<run>/` directory should follow a stabl
|
||||
"tpot_ms": { "mean": 18.24, "p50": 16.0, "p90": 28.0, "p95": 30.39, "p99": 39.57 },
|
||||
"itl_ms": { "mean": 70.29, "p50": 60.0, "p90": 110.0, "p95": 130.0, "p99": 160.0 }
|
||||
},
|
||||
"slo_status": {
|
||||
"ttft_p95_ok": true,
|
||||
"tpot_mean_ok": true,
|
||||
"overall": "✅"
|
||||
},
|
||||
"raw_requests": [
|
||||
{
|
||||
"request_id": "uuid-or-index",
|
||||
@ -214,6 +219,10 @@ The JSON file inside each `bench_results/<run>/` directory should follow a stabl
|
||||
|
||||
- `raw_requests` is optional but recommended when the JSON size is manageable. If a single run produces millions of requests, store per-request data as `raw_outputs/*.jsonl` and keep only aggregated percentiles in `results.json`.
|
||||
- Always include **P50 / P90 / P95 / P99** for TTFT, TPOT, E2E, and ITL. P95 is the primary SLO metric.
|
||||
- Include an `slo_status` object per scenario indicating whether the scenario meets the relevant SLO (e.g. S2 tier: TTFT P95 < 3000ms, TPOT mean < 50ms). Example:
|
||||
```json
|
||||
"slo_status": { "ttft_p95_ok": true, "tpot_mean_ok": true, "overall": "✅" }
|
||||
```
|
||||
- Keep field names snake_case and consistent across experiments.
|
||||
- Record hardware and engine information explicitly:
|
||||
- `accelerator` / `chip`: the accelerator family, e.g. `NVIDIA H200`, `Kunlun XPU`, `AMD MI300X`.
|
||||
@ -333,6 +342,7 @@ See `docs/H200_QUICKSTART.md` for a concrete H200 porting walkthrough.
|
||||
- [ ] `<run>/results.json` exists and follows the [Final JSON Schema](#final-json-schema).
|
||||
- [ ] `<run>/results.json` metadata records the `chip`/`accelerator` and `engine`/`backend` used.
|
||||
- [ ] `<run>/results.json` `config` records the exact server launch command/args (`server_args` or `phaseN_server_args`).
|
||||
- [ ] `<run>/results.json` each scenario records `slo_status` (overall pass/fail/partial against the relevant SLO).
|
||||
- [ ] Raw output filenames include the chip/accelerator and engine when cross-platform runs may collide.
|
||||
- [ ] `<run>/README.md` exists and documents provenance (or the report itself covers provenance).
|
||||
- [ ] Scripts either live under `experiments/<name>/` or in `scripts/` (or `scripts/<group>/`).
|
||||
|
||||
@ -78,6 +78,23 @@ def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
||||
|
||||
|
||||
def slo_status(metrics: dict) -> dict:
|
||||
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
|
||||
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
|
||||
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
mark = "✅"
|
||||
elif ttft_ok or tpot_ok:
|
||||
mark = "⚠️"
|
||||
else:
|
||||
mark = "❌"
|
||||
return {
|
||||
"ttft_p95_ok": ttft_ok,
|
||||
"tpot_mean_ok": tpot_ok,
|
||||
"overall": mark,
|
||||
}
|
||||
|
||||
|
||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
@ -96,12 +113,13 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
@ -109,9 +127,10 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n"
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@ -178,6 +197,7 @@ def main() -> None:
|
||||
"num_prompts": metrics["success"] + metrics["failed"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"slo_status": slo_status(metrics),
|
||||
"raw_file": str(jsonl_path),
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
@ -76,6 +76,23 @@ def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
||||
|
||||
|
||||
def slo_status(metrics: dict) -> dict:
|
||||
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
|
||||
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
|
||||
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
mark = "✅"
|
||||
elif ttft_ok or tpot_ok:
|
||||
mark = "⚠️"
|
||||
else:
|
||||
mark = "❌"
|
||||
return {
|
||||
"ttft_p95_ok": ttft_ok,
|
||||
"tpot_mean_ok": tpot_ok,
|
||||
"overall": mark,
|
||||
}
|
||||
|
||||
|
||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
@ -94,12 +111,13 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend vllm`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
@ -107,9 +125,10 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> N
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n"
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@ -162,6 +181,7 @@ def main() -> None:
|
||||
"num_prompts": metrics["success"] + metrics["failed"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"slo_status": slo_status(metrics),
|
||||
"raw_file": str(jsonl_path),
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user