a4e38b9e33
Move all experiments under hardware-specific folders: - experiments/h200/ : H200 GPU experiments (15 dirs) - experiments/h20/ : H20 GPU experiments (2 dirs) - experiments/p800/ : Kunlun P800 experiments (3 dirs) - experiments/pro6000/ : RTX 6000D experiments (2 dirs) This improves discoverability and keeps hardware-specific configs isolated from each other.
75 lines
3.1 KiB
Python
Executable File
75 lines
3.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate a side-by-side comparison of vLLM+MTP speculative decoding and vLLM default.
|
|
|
|
Usage:
|
|
python3 compare.py --mtp <mtp_result_root> --default <default_result_root> \
|
|
[--output comparison.md]
|
|
"""
|
|
import argparse
|
|
import json
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
def load_result(result_root: Path) -> dict:
|
|
path = result_root / "results.json"
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--mtp", type=Path, required=True)
|
|
parser.add_argument("--default", type=Path, required=True)
|
|
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
|
args = parser.parse_args()
|
|
|
|
mtp_data = load_result(args.mtp)
|
|
default_data = load_result(args.default)
|
|
|
|
by_scenario = defaultdict(dict)
|
|
for data in (mtp_data, default_data):
|
|
backend = data["metadata"]["engine"]
|
|
for s in data.get("scenarios", []):
|
|
key = s["name"]
|
|
by_scenario[key][backend] = s
|
|
|
|
with open(args.output, "w", encoding="utf-8") as f:
|
|
f.write("# vLLM+MTP speculative decoding vs vLLM default on DeepSeek-V4-Flash (H200, TP=8)\n\n")
|
|
f.write("## Summary\n\n")
|
|
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
|
|
f.write("- Hardware: 8x NVIDIA H200 143GB\n")
|
|
f.write("- Tensor Parallelism: 8\n")
|
|
f.write("- Benchmark client: `sglang.bench_serving --backend vllm`\n")
|
|
f.write("- Default: no speculative decoding\n")
|
|
f.write("- MTP: `--speculative-config '{\\\"method\\\":\\\"mtp\\\",\\\"num_speculative_tokens\\\":1}'` (official DeepSeek-V4 recipe)\n\n")
|
|
|
|
f.write("## Side-by-side results\n\n")
|
|
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) |\n")
|
|
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
|
|
|
for scenario_name in sorted(by_scenario.keys()):
|
|
for backend in ("vllm-mtp", "vllm-default"):
|
|
s = by_scenario[scenario_name].get(backend)
|
|
if s is None:
|
|
continue
|
|
cfg = s["config"]
|
|
m = s["metrics"]
|
|
f.write(
|
|
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
|
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
|
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
|
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
|
f"{m['e2e_ms']['mean']:.2f} |\n"
|
|
)
|
|
|
|
f.write("\n## Notes\n\n")
|
|
f.write("- TTFT mean/P95/P99 are the main focus for verifying MTP's impact on time-to-first-token.\n")
|
|
f.write("- Mean TPOT and E2E are included to check whether speculative decoding pays back after first token.\n")
|
|
|
|
print(f"Wrote comparison to {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|