refactor(common): remove duplicated parse/compare/warmup from existing experiments
This commit is contained in:
yy-fighting
parent
a127cf7ef0
commit
f51697aed6
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a side-by-side comparison of SGLang and vLLM results.
|
||||
|
||||
Usage:
|
||||
python3 compare.py --sglang <sglang_result_root> --vllm <vllm_result_root> \
|
||||
[--output comparison.md]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_result(result_root: Path) -> dict:
|
||||
path = result_root / "results.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
|
||||
# S2 tier targets from scripts/SLO_STANDARDS.md.
|
||||
ttft_ok = ttft_p95_ms < 3000.0
|
||||
tpot_ok = tpot_mean_ms < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
return "✅"
|
||||
if ttft_ok or tpot_ok:
|
||||
return "⚠️"
|
||||
return "❌"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--sglang", type=Path, required=True)
|
||||
parser.add_argument("--vllm", type=Path, required=True)
|
||||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||
args = parser.parse_args()
|
||||
|
||||
sglang_data = load_result(args.sglang)
|
||||
vllm_data = load_result(args.vllm)
|
||||
|
||||
by_scenario = defaultdict(dict)
|
||||
for data in (sglang_data, vllm_data):
|
||||
backend = data["metadata"]["engine"]
|
||||
for s in data.get("scenarios", []):
|
||||
key = s["name"]
|
||||
by_scenario[key][backend] = s
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n")
|
||||
f.write("## Summary\n\n")
|
||||
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write("- Hardware: 8x NVIDIA H200 143GB\n")
|
||||
f.write("- Tensor Parallelism: 8\n")
|
||||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||
f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n")
|
||||
|
||||
f.write("## Side-by-side results\n\n")
|
||||
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
|
||||
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for scenario_name in sorted(by_scenario.keys()):
|
||||
for backend in ("sglang", "vllm"):
|
||||
s = by_scenario[scenario_name].get(backend)
|
||||
if s is None:
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"])
|
||||
f.write(
|
||||
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
|
||||
)
|
||||
|
||||
f.write("\n## Notes\n\n")
|
||||
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n")
|
||||
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
||||
|
||||
print(f"Wrote comparison to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,219 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse raw sglang.bench_serving JSONL outputs for one backend.
|
||||
|
||||
Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl
|
||||
and generates results.json + report.md in the given result root.
|
||||
|
||||
Usage:
|
||||
python3 parse_backend.py <result_root> [--backend sglang|vllm]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_jsonl(path: Path) -> dict | None:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
return json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def compute_metrics(data: dict) -> dict:
|
||||
completed = data.get("completed", 0)
|
||||
total = len(data.get("input_lens", []))
|
||||
failed = total - completed if total > 0 else 0
|
||||
duration_s = data.get("duration", 0.0)
|
||||
|
||||
return {
|
||||
"success": completed,
|
||||
"failed": failed,
|
||||
"duration_s": duration_s,
|
||||
"request_throughput": data.get("request_throughput", 0.0),
|
||||
"input_token_throughput": data.get("input_throughput", 0.0),
|
||||
"output_token_throughput": data.get("output_throughput", 0.0),
|
||||
"total_token_throughput": data.get("total_throughput", 0.0),
|
||||
"total_input_tokens": data.get("total_input_tokens", 0),
|
||||
"total_output_tokens": data.get("total_output_tokens", 0),
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms", 0.0),
|
||||
"p50": data.get("median_e2e_latency_ms", 0.0),
|
||||
"p90": data.get("p90_e2e_latency_ms", 0.0),
|
||||
"p95": data.get("p95_e2e_latency_ms", 0.0),
|
||||
"p99": data.get("p99_e2e_latency_ms", 0.0),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms", 0.0),
|
||||
"p50": data.get("median_ttft_ms", 0.0),
|
||||
"p90": data.get("p90_ttft_ms", 0.0),
|
||||
"p95": data.get("p95_ttft_ms", 0.0),
|
||||
"p99": data.get("p99_ttft_ms", 0.0),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms", 0.0),
|
||||
"p50": data.get("median_tpot_ms", 0.0),
|
||||
"p90": data.get("p90_tpot_ms", 0.0),
|
||||
"p95": data.get("p95_tpot_ms", 0.0),
|
||||
"p99": data.get("p99_tpot_ms", 0.0),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms", 0.0),
|
||||
"p50": data.get("median_itl_ms", 0.0),
|
||||
"p90": data.get("p90_itl_ms", 0.0),
|
||||
"p95": data.get("p95_itl_ms", 0.0),
|
||||
"p99": data.get("p99_itl_ms", 0.0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
||||
|
||||
|
||||
def slo_status(metrics: dict) -> dict:
|
||||
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
|
||||
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
|
||||
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
mark = "✅"
|
||||
elif ttft_ok or tpot_ok:
|
||||
mark = "⚠️"
|
||||
else:
|
||||
mark = "❌"
|
||||
return {
|
||||
"ttft_p95_ok": ttft_ok,
|
||||
"tpot_mean_ok": tpot_ok,
|
||||
"overall": mark,
|
||||
}
|
||||
|
||||
|
||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"].append(scenario)
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None:
|
||||
report_path = result_root / "report.md"
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n")
|
||||
f.write(f"- Result root: `{result_root}`\n")
|
||||
f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write(f"- Backend: {backend.upper()} (TP=8)\n")
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("result_root", type=Path)
|
||||
parser.add_argument("--backend", default=None, choices=["sglang", "vllm"])
|
||||
args = parser.parse_args()
|
||||
|
||||
result_root = args.result_root
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
results_json = result_root / "results.json"
|
||||
|
||||
if not raw_dir.exists():
|
||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
||||
|
||||
backend = args.backend
|
||||
if backend is None:
|
||||
# Infer from filenames if not provided.
|
||||
for p in raw_dir.iterdir():
|
||||
if p.name.startswith("sglang_"):
|
||||
backend = "sglang"
|
||||
break
|
||||
if p.name.startswith("vllm_"):
|
||||
backend = "vllm"
|
||||
break
|
||||
if backend is None:
|
||||
raise SystemExit("Could not infer backend from raw outputs")
|
||||
|
||||
scenarios = []
|
||||
for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
|
||||
parts = jsonl_path.stem.split("_")
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
label = parts[1]
|
||||
if label == "sharegpt":
|
||||
phase = "sharegpt"
|
||||
dataset = "sharegpt"
|
||||
else:
|
||||
phase = label
|
||||
dataset = "random"
|
||||
|
||||
# The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
|
||||
try:
|
||||
concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
data = parse_jsonl(jsonl_path)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
metrics = compute_metrics(data)
|
||||
scenario = {
|
||||
"name": scenario_name(concurrency, input_len, output_len),
|
||||
"config": {
|
||||
"phase": phase,
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"dataset": dataset,
|
||||
"num_prompts": metrics["success"] + metrics["failed"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"slo_status": slo_status(metrics),
|
||||
"raw_file": str(jsonl_path),
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
if not scenarios:
|
||||
print("No benchmark outputs found to parse")
|
||||
return
|
||||
|
||||
if results_json.exists():
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"] = scenarios
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
generate_report(result_root, backend, scenarios)
|
||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send a small number of warmup requests to a running backend.
|
||||
|
||||
Uses sglang.bench_serving with a single request so that the same code path
|
||||
(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real
|
||||
benchmark begins. Discards the output.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None:
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp:
|
||||
cmd = [
|
||||
str(env_python),
|
||||
"-m",
|
||||
"sglang.bench_serving",
|
||||
"--backend",
|
||||
backend,
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--dataset-name",
|
||||
"random",
|
||||
"--random-input-len",
|
||||
str(input_len),
|
||||
"--random-output-len",
|
||||
str(output_len),
|
||||
"--num-prompts",
|
||||
str(num),
|
||||
"--max-concurrency",
|
||||
"1",
|
||||
"--request-rate",
|
||||
"10000",
|
||||
"--output-file",
|
||||
tmp.name,
|
||||
"--output-details",
|
||||
]
|
||||
print(f"[warmup] {' '.join(cmd)}", flush=True)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print("[warmup] FAILED", file=sys.stderr)
|
||||
print(result.stdout, file=sys.stderr)
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Warmup a serving backend.")
|
||||
parser.add_argument("--backend", required=True, choices=["sglang", "vllm"])
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--input-len", type=int, required=True)
|
||||
parser.add_argument("--output-len", type=int, required=True)
|
||||
parser.add_argument("--num", type=int, default=1)
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_warmup(
|
||||
backend=args.backend,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
input_len=args.input_len,
|
||||
output_len=args.output_len,
|
||||
num=args.num,
|
||||
env_python=Path(args.env_python),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a side-by-side comparison of SGLang and vLLM results.
|
||||
|
||||
Usage:
|
||||
python3 compare.py --sglang <sglang_result_root> --vllm <vllm_result_root> \
|
||||
[--output comparison.md]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_result(result_root: Path) -> dict:
|
||||
path = result_root / "results.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
|
||||
# S2 tier targets from scripts/SLO_STANDARDS.md.
|
||||
ttft_ok = ttft_p95_ms < 3000.0
|
||||
tpot_ok = tpot_mean_ms < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
return "✅"
|
||||
if ttft_ok or tpot_ok:
|
||||
return "⚠️"
|
||||
return "❌"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--sglang", type=Path, required=True)
|
||||
parser.add_argument("--vllm", type=Path, required=True)
|
||||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||
args = parser.parse_args()
|
||||
|
||||
sglang_data = load_result(args.sglang)
|
||||
vllm_data = load_result(args.vllm)
|
||||
|
||||
by_scenario = defaultdict(dict)
|
||||
for data in (sglang_data, vllm_data):
|
||||
backend = data["metadata"]["engine"]
|
||||
for s in data.get("scenarios", []):
|
||||
key = s["name"]
|
||||
by_scenario[key][backend] = s
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n")
|
||||
f.write("## Summary\n\n")
|
||||
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write("- Hardware: 8x NVIDIA H200 143GB\n")
|
||||
f.write("- Tensor Parallelism: 8\n")
|
||||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||
f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n")
|
||||
|
||||
f.write("## Side-by-side results\n\n")
|
||||
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
|
||||
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for scenario_name in sorted(by_scenario.keys()):
|
||||
for backend in ("sglang", "vllm"):
|
||||
s = by_scenario[scenario_name].get(backend)
|
||||
if s is None:
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"])
|
||||
f.write(
|
||||
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
|
||||
)
|
||||
|
||||
f.write("\n## Notes\n\n")
|
||||
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n")
|
||||
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
||||
|
||||
print(f"Wrote comparison to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,219 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse raw sglang.bench_serving JSONL outputs for one backend.
|
||||
|
||||
Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl
|
||||
and generates results.json + report.md in the given result root.
|
||||
|
||||
Usage:
|
||||
python3 parse_backend.py <result_root> [--backend sglang|vllm]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_jsonl(path: Path) -> dict | None:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
return json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def compute_metrics(data: dict) -> dict:
|
||||
completed = data.get("completed", 0)
|
||||
total = len(data.get("input_lens", []))
|
||||
failed = total - completed if total > 0 else 0
|
||||
duration_s = data.get("duration", 0.0)
|
||||
|
||||
return {
|
||||
"success": completed,
|
||||
"failed": failed,
|
||||
"duration_s": duration_s,
|
||||
"request_throughput": data.get("request_throughput", 0.0),
|
||||
"input_token_throughput": data.get("input_throughput", 0.0),
|
||||
"output_token_throughput": data.get("output_throughput", 0.0),
|
||||
"total_token_throughput": data.get("total_throughput", 0.0),
|
||||
"total_input_tokens": data.get("total_input_tokens", 0),
|
||||
"total_output_tokens": data.get("total_output_tokens", 0),
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms", 0.0),
|
||||
"p50": data.get("median_e2e_latency_ms", 0.0),
|
||||
"p90": data.get("p90_e2e_latency_ms", 0.0),
|
||||
"p95": data.get("p95_e2e_latency_ms", 0.0),
|
||||
"p99": data.get("p99_e2e_latency_ms", 0.0),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms", 0.0),
|
||||
"p50": data.get("median_ttft_ms", 0.0),
|
||||
"p90": data.get("p90_ttft_ms", 0.0),
|
||||
"p95": data.get("p95_ttft_ms", 0.0),
|
||||
"p99": data.get("p99_ttft_ms", 0.0),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms", 0.0),
|
||||
"p50": data.get("median_tpot_ms", 0.0),
|
||||
"p90": data.get("p90_tpot_ms", 0.0),
|
||||
"p95": data.get("p95_tpot_ms", 0.0),
|
||||
"p99": data.get("p99_tpot_ms", 0.0),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms", 0.0),
|
||||
"p50": data.get("median_itl_ms", 0.0),
|
||||
"p90": data.get("p90_itl_ms", 0.0),
|
||||
"p95": data.get("p95_itl_ms", 0.0),
|
||||
"p99": data.get("p99_itl_ms", 0.0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
||||
|
||||
|
||||
def slo_status(metrics: dict) -> dict:
|
||||
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
|
||||
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
|
||||
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
mark = "✅"
|
||||
elif ttft_ok or tpot_ok:
|
||||
mark = "⚠️"
|
||||
else:
|
||||
mark = "❌"
|
||||
return {
|
||||
"ttft_p95_ok": ttft_ok,
|
||||
"tpot_mean_ok": tpot_ok,
|
||||
"overall": mark,
|
||||
}
|
||||
|
||||
|
||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"].append(scenario)
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None:
|
||||
report_path = result_root / "report.md"
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n")
|
||||
f.write(f"- Result root: `{result_root}`\n")
|
||||
f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write(f"- Backend: {backend.upper()} (TP=8)\n")
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("result_root", type=Path)
|
||||
parser.add_argument("--backend", default=None, choices=["sglang", "vllm"])
|
||||
args = parser.parse_args()
|
||||
|
||||
result_root = args.result_root
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
results_json = result_root / "results.json"
|
||||
|
||||
if not raw_dir.exists():
|
||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
||||
|
||||
backend = args.backend
|
||||
if backend is None:
|
||||
# Infer from filenames if not provided.
|
||||
for p in raw_dir.iterdir():
|
||||
if p.name.startswith("sglang_"):
|
||||
backend = "sglang"
|
||||
break
|
||||
if p.name.startswith("vllm_"):
|
||||
backend = "vllm"
|
||||
break
|
||||
if backend is None:
|
||||
raise SystemExit("Could not infer backend from raw outputs")
|
||||
|
||||
scenarios = []
|
||||
for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
|
||||
parts = jsonl_path.stem.split("_")
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
label = parts[1]
|
||||
if label == "sharegpt":
|
||||
phase = "sharegpt"
|
||||
dataset = "sharegpt"
|
||||
else:
|
||||
phase = label
|
||||
dataset = "random"
|
||||
|
||||
# The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
|
||||
try:
|
||||
concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
data = parse_jsonl(jsonl_path)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
metrics = compute_metrics(data)
|
||||
scenario = {
|
||||
"name": scenario_name(concurrency, input_len, output_len),
|
||||
"config": {
|
||||
"phase": phase,
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"dataset": dataset,
|
||||
"num_prompts": metrics["success"] + metrics["failed"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"slo_status": slo_status(metrics),
|
||||
"raw_file": str(jsonl_path),
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
if not scenarios:
|
||||
print("No benchmark outputs found to parse")
|
||||
return
|
||||
|
||||
if results_json.exists():
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"] = scenarios
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
generate_report(result_root, backend, scenarios)
|
||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send a small number of warmup requests to a running backend.
|
||||
|
||||
Uses sglang.bench_serving with a single request so that the same code path
|
||||
(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real
|
||||
benchmark begins. Discards the output.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None:
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp:
|
||||
cmd = [
|
||||
str(env_python),
|
||||
"-m",
|
||||
"sglang.bench_serving",
|
||||
"--backend",
|
||||
backend,
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--dataset-name",
|
||||
"random",
|
||||
"--random-input-len",
|
||||
str(input_len),
|
||||
"--random-output-len",
|
||||
str(output_len),
|
||||
"--num-prompts",
|
||||
str(num),
|
||||
"--max-concurrency",
|
||||
"1",
|
||||
"--request-rate",
|
||||
"10000",
|
||||
"--output-file",
|
||||
tmp.name,
|
||||
"--output-details",
|
||||
]
|
||||
print(f"[warmup] {' '.join(cmd)}", flush=True)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print("[warmup] FAILED", file=sys.stderr)
|
||||
print(result.stdout, file=sys.stderr)
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Warmup a serving backend.")
|
||||
parser.add_argument("--backend", required=True, choices=["sglang", "vllm"])
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--input-len", type=int, required=True)
|
||||
parser.add_argument("--output-len", type=int, required=True)
|
||||
parser.add_argument("--num", type=int, default=1)
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_warmup(
|
||||
backend=args.backend,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
input_len=args.input_len,
|
||||
output_len=args.output_len,
|
||||
num=args.num,
|
||||
env_python=Path(args.env_python),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a side-by-side comparison of SGLang and vLLM results.
|
||||
|
||||
Usage:
|
||||
python3 compare.py --sglang <sglang_result_root> --vllm <vllm_result_root> \
|
||||
[--output comparison.md]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_result(result_root: Path) -> dict:
|
||||
path = result_root / "results.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
|
||||
# S2 tier targets from scripts/SLO_STANDARDS.md.
|
||||
ttft_ok = ttft_p95_ms < 3000.0
|
||||
tpot_ok = tpot_mean_ms < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
return "✅"
|
||||
if ttft_ok or tpot_ok:
|
||||
return "⚠️"
|
||||
return "❌"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--sglang", type=Path, required=True)
|
||||
parser.add_argument("--vllm", type=Path, required=True)
|
||||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||
args = parser.parse_args()
|
||||
|
||||
sglang_data = load_result(args.sglang)
|
||||
vllm_data = load_result(args.vllm)
|
||||
|
||||
by_scenario = defaultdict(dict)
|
||||
for data in (sglang_data, vllm_data):
|
||||
backend = data["metadata"]["engine"]
|
||||
for s in data.get("scenarios", []):
|
||||
key = s["name"]
|
||||
by_scenario[key][backend] = s
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n")
|
||||
f.write("## Summary\n\n")
|
||||
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write("- Hardware: 8x NVIDIA H200 143GB\n")
|
||||
f.write("- Tensor Parallelism: 8\n")
|
||||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||
f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n")
|
||||
|
||||
f.write("## Side-by-side results\n\n")
|
||||
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
|
||||
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for scenario_name in sorted(by_scenario.keys()):
|
||||
for backend in ("sglang", "vllm"):
|
||||
s = by_scenario[scenario_name].get(backend)
|
||||
if s is None:
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"])
|
||||
f.write(
|
||||
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
|
||||
)
|
||||
|
||||
f.write("\n## Notes\n\n")
|
||||
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n")
|
||||
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
||||
|
||||
print(f"Wrote comparison to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,221 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse raw sglang.bench_serving JSONL outputs for one backend.
|
||||
|
||||
Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl
|
||||
and generates results.json + report.md in the given result root.
|
||||
|
||||
Usage:
|
||||
python3 parse_backend.py <result_root> [--backend sglang|vllm]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_jsonl(path: Path) -> dict | None:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
return json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def compute_metrics(data: dict) -> dict:
|
||||
completed = data.get("completed", 0)
|
||||
total = len(data.get("input_lens", []))
|
||||
failed = total - completed if total > 0 else 0
|
||||
duration_s = data.get("duration", 0.0)
|
||||
|
||||
return {
|
||||
"success": completed,
|
||||
"failed": failed,
|
||||
"duration_s": duration_s,
|
||||
"request_throughput": data.get("request_throughput", 0.0),
|
||||
"input_token_throughput": data.get("input_throughput", 0.0),
|
||||
"output_token_throughput": data.get("output_throughput", 0.0),
|
||||
"total_token_throughput": data.get("total_throughput", 0.0),
|
||||
"total_input_tokens": data.get("total_input_tokens", 0),
|
||||
"total_output_tokens": data.get("total_output_tokens", 0),
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms", 0.0),
|
||||
"p50": data.get("median_e2e_latency_ms", 0.0),
|
||||
"p90": data.get("p90_e2e_latency_ms", 0.0),
|
||||
"p95": data.get("p95_e2e_latency_ms", 0.0),
|
||||
"p99": data.get("p99_e2e_latency_ms", 0.0),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms", 0.0),
|
||||
"p50": data.get("median_ttft_ms", 0.0),
|
||||
"p90": data.get("p90_ttft_ms", 0.0),
|
||||
"p95": data.get("p95_ttft_ms", 0.0),
|
||||
"p99": data.get("p99_ttft_ms", 0.0),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms", 0.0),
|
||||
"p50": data.get("median_tpot_ms", 0.0),
|
||||
"p90": data.get("p90_tpot_ms", 0.0),
|
||||
"p95": data.get("p95_tpot_ms", 0.0),
|
||||
"p99": data.get("p99_tpot_ms", 0.0),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms", 0.0),
|
||||
"p50": data.get("median_itl_ms", 0.0),
|
||||
"p90": data.get("p90_itl_ms", 0.0),
|
||||
"p95": data.get("p95_itl_ms", 0.0),
|
||||
"p99": data.get("p99_itl_ms", 0.0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
||||
|
||||
|
||||
def slo_status(metrics: dict) -> dict:
|
||||
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
|
||||
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
|
||||
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
|
||||
if ttft_ok and tpot_ok:
|
||||
mark = "✅"
|
||||
elif ttft_ok or tpot_ok:
|
||||
mark = "⚠️"
|
||||
else:
|
||||
mark = "❌"
|
||||
return {
|
||||
"ttft_p95_ok": ttft_ok,
|
||||
"tpot_mean_ok": tpot_ok,
|
||||
"overall": mark,
|
||||
}
|
||||
|
||||
|
||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"].append(scenario)
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None:
|
||||
report_path = result_root / "report.md"
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n")
|
||||
f.write(f"- Result root: `{result_root}`\n")
|
||||
f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
||||
f.write(f"- Backend: {backend.upper()} (TP=8)\n")
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("result_root", type=Path)
|
||||
parser.add_argument("--backend", default=None, choices=["sglang", "vllm"])
|
||||
args = parser.parse_args()
|
||||
|
||||
result_root = args.result_root
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
results_json = result_root / "results.json"
|
||||
|
||||
if not raw_dir.exists():
|
||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
||||
|
||||
backend = args.backend
|
||||
if backend is None:
|
||||
# Infer from filenames if not provided.
|
||||
for p in raw_dir.iterdir():
|
||||
if p.name.startswith("sglang_"):
|
||||
backend = "sglang"
|
||||
break
|
||||
if p.name.startswith("vllm_"):
|
||||
backend = "vllm"
|
||||
break
|
||||
if backend is None:
|
||||
raise SystemExit("Could not infer backend from raw outputs")
|
||||
|
||||
scenarios = []
|
||||
for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
|
||||
parts = jsonl_path.stem.split("_")
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
label = parts[1]
|
||||
if label.startswith("phase"):
|
||||
phase = label
|
||||
dataset = "random"
|
||||
elif label == "sharegpt":
|
||||
phase = "sharegpt"
|
||||
dataset = "sharegpt"
|
||||
else:
|
||||
continue
|
||||
|
||||
# The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
|
||||
try:
|
||||
concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
data = parse_jsonl(jsonl_path)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
metrics = compute_metrics(data)
|
||||
scenario = {
|
||||
"name": scenario_name(concurrency, input_len, output_len),
|
||||
"config": {
|
||||
"phase": phase,
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"dataset": dataset,
|
||||
"num_prompts": metrics["success"] + metrics["failed"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"slo_status": slo_status(metrics),
|
||||
"raw_file": str(jsonl_path),
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
if not scenarios:
|
||||
print("No benchmark outputs found to parse")
|
||||
return
|
||||
|
||||
if results_json.exists():
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"] = scenarios
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
generate_report(result_root, backend, scenarios)
|
||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send a small number of warmup requests to a running backend.
|
||||
|
||||
Uses sglang.bench_serving with a single request so that the same code path
|
||||
(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real
|
||||
benchmark begins. Discards the output.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None:
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp:
|
||||
cmd = [
|
||||
str(env_python),
|
||||
"-m",
|
||||
"sglang.bench_serving",
|
||||
"--backend",
|
||||
backend,
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--dataset-name",
|
||||
"random",
|
||||
"--random-input-len",
|
||||
str(input_len),
|
||||
"--random-output-len",
|
||||
str(output_len),
|
||||
"--num-prompts",
|
||||
str(num),
|
||||
"--max-concurrency",
|
||||
"1",
|
||||
"--request-rate",
|
||||
"10000",
|
||||
"--output-file",
|
||||
tmp.name,
|
||||
"--output-details",
|
||||
]
|
||||
print(f"[warmup] {' '.join(cmd)}", flush=True)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print("[warmup] FAILED", file=sys.stderr)
|
||||
print(result.stdout, file=sys.stderr)
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Warmup a serving backend.")
|
||||
parser.add_argument("--backend", required=True, choices=["sglang", "vllm"])
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--input-len", type=int, required=True)
|
||||
parser.add_argument("--output-len", type=int, required=True)
|
||||
parser.add_argument("--num", type=int, default=1)
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_warmup(
|
||||
backend=args.backend,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
input_len=args.input_len,
|
||||
output_len=args.output_len,
|
||||
num=args.num,
|
||||
env_python=Path(args.env_python),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user