a4e38b9e33
Move all experiments under hardware-specific folders: - experiments/h200/ : H200 GPU experiments (15 dirs) - experiments/h20/ : H20 GPU experiments (2 dirs) - experiments/p800/ : Kunlun P800 experiments (3 dirs) - experiments/pro6000/ : RTX 6000D experiments (2 dirs) This improves discoverability and keeps hardware-specific configs isolated from each other.
164 lines
6.9 KiB
Python
164 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
||
"""Cross TP×DP configuration comparison for dsv4_h20_vllm_tp_dp_matrix.
|
||
|
||
Usage:
|
||
python3 compare.py --run-root results/<run_id> [--output comparison.md]
|
||
"""
|
||
import argparse
|
||
import json
|
||
import re
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
|
||
def load_result(result_root: Path) -> dict:
|
||
path = result_root / "results.json"
|
||
with open(path, "r", encoding="utf-8") as f:
|
||
return json.load(f)
|
||
|
||
|
||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
|
||
ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str:
|
||
ttft_ok = ttft_p95_ms < ttft_limit_ms
|
||
tpot_ok = tpot_mean_ms < tpot_limit_ms
|
||
if ttft_ok and tpot_ok:
|
||
return "PASS"
|
||
if ttft_ok or tpot_ok:
|
||
return "PARTIAL"
|
||
return "FAIL"
|
||
|
||
|
||
def gpu_memory_str(gpu: dict | None) -> str:
|
||
if not gpu:
|
||
return "-"
|
||
peak = gpu.get("peak_used_mb", 0)
|
||
total = gpu.get("memory_total_mb", 0)
|
||
if total:
|
||
return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)"
|
||
return f"{peak:.0f}"
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--run-root", type=Path, required=True)
|
||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||
parser.add_argument("--ttft-limit", type=float, default=3000.0)
|
||
parser.add_argument("--tpot-limit", type=float, default=50.0)
|
||
args = parser.parse_args()
|
||
|
||
# Discover configurations: tp*_dp* directories.
|
||
configs = []
|
||
for subdir in sorted(args.run_root.iterdir()):
|
||
if not subdir.is_dir():
|
||
continue
|
||
name = subdir.name
|
||
if not (name.startswith("tp") and "_dp" in name):
|
||
continue
|
||
results_json = subdir / "results.json"
|
||
if not results_json.exists():
|
||
continue
|
||
configs.append((name, load_result(subdir)))
|
||
|
||
if not configs:
|
||
print(f"No tp*_dp* results found under {args.run_root}")
|
||
return
|
||
|
||
model = configs[0][1].get("metadata", {}).get("model", "unknown")
|
||
hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown")
|
||
|
||
# Group by scenario name.
|
||
by_scenario: dict[str, dict[str, dict]] = defaultdict(dict)
|
||
skipped: dict[str, dict[str, str]] = defaultdict(dict)
|
||
for label, data in configs:
|
||
for s in data.get("scenarios", []):
|
||
key = s["name"]
|
||
if s.get("status") == "skipped_oom":
|
||
skipped[key][label] = s.get("note", "skipped")
|
||
else:
|
||
by_scenario[key][label] = s
|
||
|
||
with open(args.output, "w", encoding="utf-8") as f:
|
||
f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n")
|
||
f.write("## Summary\n\n")
|
||
f.write(f"- Model: `{model}`\n")
|
||
f.write(f"- Hardware: {hardware}\n")
|
||
f.write("- Backend: vLLM (Docker)\n")
|
||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
|
||
|
||
# Configuration overview.
|
||
f.write("### Configurations\n\n")
|
||
f.write("| Config | TP | DP | GPUs/replica | Notes |\n")
|
||
f.write("|---|---:|---:|---:|---|\n")
|
||
for label, data in configs:
|
||
cfg = data.get("config", {})
|
||
tp = cfg.get("tp", "?")
|
||
dp = cfg.get("dp", "?")
|
||
f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n")
|
||
f.write("\n")
|
||
|
||
# Side-by-side table.
|
||
f.write("## Side-by-side results\n\n")
|
||
headers = [
|
||
"Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s",
|
||
"TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)",
|
||
"TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO"
|
||
]
|
||
f.write("| " + " | ".join(headers) + " |\n")
|
||
f.write("|" + "|".join(["---"] * len(headers)) + "|\n")
|
||
|
||
for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))):
|
||
_, isl, osl = re.findall(r"\d+", scenario_name)
|
||
# cfg_part not used; just for readability.
|
||
for label, data in configs:
|
||
s = by_scenario[scenario_name].get(label)
|
||
if s is None:
|
||
if scenario_name in skipped and label in skipped[scenario_name]:
|
||
note = skipped[scenario_name][label]
|
||
f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n")
|
||
continue
|
||
cfg = s["config"]
|
||
m = s["metrics"]
|
||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||
gpu = m.get("gpu_memory")
|
||
f.write(
|
||
f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | "
|
||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||
f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n"
|
||
)
|
||
|
||
# Best throughput per ISL/OSL.
|
||
f.write("\n## Best throughput per (ISL, OSL)\n\n")
|
||
f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n")
|
||
f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n")
|
||
best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {}
|
||
for scenario_name, backends in by_scenario.items():
|
||
_, isl, osl = re.findall(r"\d+", scenario_name)
|
||
isl_i, osl_i = int(isl), int(osl)
|
||
for label, s in backends.items():
|
||
m = s["metrics"]
|
||
out_tok = m["output_token_throughput"]
|
||
if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]:
|
||
best_by_shape[(isl_i, osl_i)] = (out_tok, label, s)
|
||
for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()):
|
||
m = s["metrics"]
|
||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||
f.write(
|
||
f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | "
|
||
f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n"
|
||
)
|
||
|
||
f.write("\n## Notes\n\n")
|
||
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
|
||
f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n")
|
||
f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
|
||
f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")
|
||
|
||
print(f"Wrote comparison to {args.output}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|