diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/comparison.md b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/comparison.md new file mode 100644 index 0000000..dfc98f0 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/comparison.md @@ -0,0 +1,41 @@ +# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8) + +## Summary + +- Model: `/data/models/DeepSeek-V4-Flash` +- Hardware: 8x NVIDIA H200 143GB +- Tensor Parallelism: 8 +- Benchmark client: `sglang.bench_serving` +- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms) + +## Side-by-side results + +| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | sglang | 128 | 512 | 256 | 27.96 | 3691.88 | 1046.81 | 1048.77 | 19.84 | 30.70 | 78.39 | 4494.67 | ✅ | +| c128_i512_o256 | vllm | 128 | 512 | 256 | 1.82 | 240.43 | 65775.64 | 68346.91 | 7.51 | 8.88 | 9.95 | 69371.23 | ⚠️ | +| c1_i131072_o1024 | sglang | 1 | 131072 | 1024 | 0.21 | 117.45 | 610.97 | 623.90 | 7.64 | 7.71 | 7.72 | 7216.79 | ✅ | +| c1_i131072_o1024 | vllm | 1 | 131072 | 1024 | 0.24 | 133.99 | 549.11 | 565.06 | 6.79 | 6.84 | 6.84 | 6410.99 | ✅ | +| c1_i200000_o1024 | sglang | 1 | 200000 | 1024 | 0.25 | 109.00 | 619.26 | 619.26 | 7.72 | 7.72 | 7.72 | 3976.45 | ✅ | +| c1_i200000_o1024 | vllm | 1 | 200000 | 1024 | 0.28 | 121.52 | 608.85 | 608.85 | 6.84 | 6.84 | 6.84 | 3584.28 | ✅ | +| c1_i32768_o1024 | sglang | 1 | 32768 | 1024 | 0.32 | 107.76 | 1088.23 | 1185.18 | 7.55 | 7.58 | 7.58 | 4928.38 | ✅ | +| c1_i32768_o1024 | vllm | 1 | 32768 | 1024 | 0.36 | 120.28 | 1007.75 | 1099.08 | 6.70 | 6.76 | 6.76 | 4391.50 | ✅ | +| c1_i32768_o256 | sglang | 1 | 32768 | 256 | 0.48 | 122.41 | 225.25 | 228.79 | 7.43 | 7.44 | 7.44 | 2126.29 | ✅ | +| c1_i32768_o256 | vllm | 1 | 32768 | 256 | 0.53 | 135.36 | 319.19 | 332.05 | 6.66 | 6.66 | 6.67 | 2026.78 | ✅ | +| c1_i4000_o512 | sglang | 1 | 4000 | 512 | 0.43 | 121.90 | 245.14 | 250.72 | 7.50 | 7.58 | 7.59 | 4062.79 | ✅ | +| c1_i4000_o512 | vllm | 1 | 4000 | 512 | 0.49 | 138.92 | 195.78 | 198.56 | 6.68 | 6.73 | 6.73 | 3604.09 | ✅ | +| c1_i512_o256 | sglang | 1 | 512 | 256 | 0.77 | 112.74 | 228.17 | 229.05 | 7.45 | 7.47 | 7.62 | 2105.10 | ✅ | +| c1_i512_o256 | vllm | 1 | 512 | 256 | 0.92 | 133.24 | 188.08 | 196.56 | 6.67 | 6.71 | 6.73 | 1877.61 | ✅ | +| c1_i65536_o1024 | sglang | 1 | 65536 | 1024 | 0.43 | 97.35 | 1391.18 | 1526.74 | 7.56 | 7.62 | 7.63 | 5333.41 | ✅ | +| c1_i65536_o1024 | vllm | 1 | 65536 | 1024 | 0.50 | 113.04 | 1199.14 | 1320.28 | 6.11 | 6.66 | 6.67 | 4685.90 | ✅ | +| c32_i32768_o256 | sglang | 32 | 32768 | 256 | 9.11 | 2331.04 | 894.53 | 897.13 | 11.22 | 11.79 | 12.38 | 3706.48 | ✅ | +| c32_i32768_o256 | vllm | 32 | 32768 | 256 | 1.03 | 263.21 | 29346.73 | 29758.65 | 7.18 | 7.46 | 7.48 | 31587.45 | ⚠️ | +| c32_i4000_o512 | sglang | 32 | 4000 | 512 | 4.60 | 1298.23 | 647.63 | 650.43 | 19.83 | 25.78 | 27.20 | 11422.18 | ✅ | +| c32_i4000_o512 | vllm | 32 | 4000 | 512 | 0.93 | 263.43 | 33396.12 | 33949.08 | 7.20 | 7.61 | 8.33 | 36115.49 | ⚠️ | +| c32_i512_o256 | sglang | 32 | 512 | 256 | 5.71 | 753.27 | 448.83 | 573.80 | 38.34 | 53.13 | 76.13 | 11783.40 | ✅ | +| c32_i512_o256 | vllm | 32 | 512 | 256 | 1.81 | 239.27 | 17481.79 | 18082.76 | 7.54 | 9.09 | 9.97 | 19259.08 | ⚠️ | + +## Notes + +- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`). +- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out. diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/report.md b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/report.md new file mode 100644 index 0000000..3d2745e --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/report.md @@ -0,0 +1,25 @@ +# H200 SGLANG Comparison Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: SGLANG (TP=8) +- Benchmark client: `sglang.bench_serving --backend sglang` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | phase1 | 128 | 512 | 256 | 4.58 | 128 | 27.96 | 7700.07 | 3691.88 | 11391.95 | 884.51 | 1046.81 | 1048.77 | 19.84 | 30.70 | 78.39 | 3018.50 | 4438.56 | 4494.67 | ✅ | +| c1_i4000_o512 | phase1 | 1 | 4000 | 512 | 73.90 | 32 | 0.43 | 879.55 | 121.90 | 1001.45 | 206.80 | 245.14 | 250.72 | 7.50 | 7.58 | 7.59 | 2307.73 | 4000.18 | 4062.79 | ✅ | +| c1_i512_o256 | phase1 | 1 | 512 | 256 | 41.31 | 32 | 0.77 | 197.79 | 112.74 | 310.53 | 213.56 | 228.17 | 229.05 | 7.45 | 7.47 | 7.62 | 1289.17 | 2101.38 | 2105.10 | ✅ | +| c32_i4000_o512 | phase1 | 32 | 4000 | 512 | 13.92 | 64 | 4.60 | 9658.35 | 1298.23 | 10956.57 | 460.54 | 647.63 | 650.43 | 19.83 | 25.78 | 27.20 | 5948.45 | 10850.62 | 11422.18 | ✅ | +| c32_i512_o256 | phase1 | 32 | 512 | 256 | 22.44 | 128 | 5.71 | 1571.09 | 753.27 | 2324.36 | 336.61 | 448.83 | 573.80 | 38.34 | 53.13 | 76.13 | 5292.31 | 11043.01 | 11783.40 | ✅ | +| c1_i131072_o1024 | phase2 | 1 | 131072 | 1024 | 9.64 | 2 | 0.21 | 14292.96 | 117.45 | 14410.41 | 465.44 | 610.97 | 623.90 | 7.64 | 7.71 | 7.72 | 4805.89 | 7019.98 | 7216.79 | ✅ | +| c1_i200000_o1024 | phase2 | 1 | 200000 | 1024 | 4.00 | 1 | 0.25 | 30489.99 | 109.00 | 30598.99 | 619.26 | 619.26 | 619.26 | 7.72 | 7.72 | 7.72 | 3976.45 | 3976.45 | 3976.45 | ✅ | +| c1_i32768_o1024 | phase2 | 1 | 32768 | 1024 | 24.78 | 8 | 0.32 | 4856.52 | 107.76 | 4964.28 | 582.17 | 1088.23 | 1185.18 | 7.55 | 7.58 | 7.58 | 3093.22 | 4747.05 | 4928.38 | ✅ | +| c1_i65536_o1024 | phase2 | 1 | 65536 | 1024 | 9.26 | 4 | 0.43 | 12018.95 | 97.35 | 12116.30 | 627.00 | 1391.18 | 1526.74 | 7.56 | 7.62 | 7.63 | 2307.18 | 4885.48 | 5333.41 | ✅ | +| c1_i32768_o256 | sharegpt | 1 | 32768 | 256 | 66.92 | 32 | 0.48 | 148.71 | 122.41 | 271.12 | 194.08 | 225.25 | 228.79 | 7.43 | 7.44 | 7.44 | 2089.53 | 2121.84 | 2126.29 | ✅ | +| c32_i32768_o256 | sharegpt | 32 | 32768 | 256 | 14.06 | 128 | 9.11 | 3273.05 | 2331.04 | 5604.09 | 629.65 | 894.53 | 897.13 | 11.22 | 11.79 | 12.38 | 3490.74 | 3703.17 | 3706.48 | ✅ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/results.json b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/results.json new file mode 100644 index 0000000..9fce60b --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/results.json @@ -0,0 +1,646 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang_vs_vllm_sglang", + "run_id": "20260708-083346", + "timestamp": "2026-07-08T08:44:53+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "0135a5f", + "git_dirty": "dirty", + "description": "H200 sglang TP=8 comparison benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": "sglang", + "server_start_script": "experiments/dsv4_h200_sglang_vs_vllm/start_sglang.sh", + "phase1_server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 32768 --max-running-requests 256 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30006", + "phase2_server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 210000 --max-running-requests 2 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30006" + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "phase": "phase1", + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 4.577618950992473, + "request_throughput": 27.96213519962126, + "input_token_throughput": 7700.072980595705, + "output_token_throughput": 3691.875663074995, + "total_token_throughput": 11391.9486436707, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 3018.4967098989546, + "p50": 3125.7881020064815, + "p90": 4381.1214389003, + "p95": 4438.557288451557, + "p99": 4494.668244577188 + }, + "ttft_ms": { + "mean": 884.510967406527, + "p50": 844.3955024995375, + "p90": 1043.815255498339, + "p95": 1046.8067594003514, + "p99": 1048.7736633588793 + }, + "tpot_ms": { + "mean": 19.84203271438608, + "p50": 16.453881444586266, + "p90": 24.10356060360325, + "p95": 30.700567981948517, + "p99": 78.38535120434243 + }, + "itl_ms": { + "mean": 16.286870400989425, + "p50": 15.138339003897272, + "p90": 17.414973000995815, + "p95": 18.159660998207983, + "p99": 21.124977292492986 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase1_0708_128_512_256.jsonl" + }, + { + "name": "c1_i4000_o512", + "config": { + "phase": "phase1", + "concurrency": 1, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 73.90364093599783, + "request_throughput": 0.4329962582995431, + "input_token_throughput": 879.5507119370906, + "output_token_throughput": 121.90197784439323, + "total_token_throughput": 1001.4526897814839, + "total_input_tokens": 65002, + "total_output_tokens": 9009, + "e2e_ms": { + "mean": 2307.7325920953626, + "p50": 2345.350652009074, + "p90": 3828.394395999203, + "p95": 4000.1761078070554, + "p99": 4062.7909016507333 + }, + "ttft_ms": { + "mean": 206.7965900314448, + "p50": 201.0829190039658, + "p90": 240.6535267873551, + "p95": 245.1369363981939, + "p99": 250.71827917563493 + }, + "tpot_ms": { + "mean": 7.498154431018657, + "p50": 7.5119136901229195, + "p90": 7.523741559697524, + "p95": 7.584569910587951, + "p99": 7.585661201602696 + }, + "itl_ms": { + "mean": 7.48895231012723, + "p50": 7.482982007786632, + "p90": 7.666864397469908, + "p95": 7.742873395909555, + "p99": 8.035328361438587 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase1_0708_1_4000_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "phase": "phase1", + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 41.30654629699711, + "request_throughput": 0.7746956080500568, + "input_token_throughput": 197.7894724302801, + "output_token_throughput": 112.74242020903483, + "total_token_throughput": 310.53189263931495, + "total_input_tokens": 8170, + "total_output_tokens": 4657, + "e2e_ms": { + "mean": 1289.1700838772522, + "p50": 1492.7323750016512, + "p90": 2059.0501777929603, + "p95": 2101.384461154521, + "p99": 2105.098063904734 + }, + "ttft_ms": { + "mean": 213.55524078308008, + "p50": 210.26033050293336, + "p90": 221.74607979977736, + "p95": 228.1717136989755, + "p99": 229.0484117741289 + }, + "tpot_ms": { + "mean": 7.450517788338341, + "p50": 7.44191210684248, + "p90": 7.463146393434216, + "p95": 7.473324023989595, + "p99": 7.617010675142713 + }, + "itl_ms": { + "mean": 7.441748419027726, + "p50": 7.430778990965337, + "p90": 7.689515000674874, + "p95": 7.785665598930791, + "p99": 8.106428646715361 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase1_0708_1_512_256.jsonl" + }, + { + "name": "c32_i4000_o512", + "config": { + "phase": "phase1", + "concurrency": 32, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 64 + }, + "metrics": { + "success": 64, + "failed": 0, + "duration_s": 13.918221335989074, + "request_throughput": 4.598288707660644, + "input_token_throughput": 9658.346189135897, + "output_token_throughput": 1298.2262290425028, + "total_token_throughput": 10956.572418178399, + "total_input_tokens": 134427, + "total_output_tokens": 18069, + "e2e_ms": { + "mean": 5948.451856109614, + "p50": 5865.095270004531, + "p90": 10548.594036900613, + "p95": 10850.61511455715, + "p99": 11422.181814811192 + }, + "ttft_ms": { + "mean": 460.54040595277, + "p50": 507.2956724980031, + "p90": 644.1426863952074, + "p95": 647.6346344941702, + "p99": 650.4269159617252 + }, + "tpot_ms": { + "mean": 19.83145335299758, + "p50": 21.345301129403698, + "p90": 24.627898798156398, + "p95": 25.783460396012394, + "p99": 27.203415422495656 + }, + "itl_ms": { + "mean": 19.507043877033254, + "p50": 10.689528993680142, + "p90": 20.828651206102233, + "p95": 87.12114800291602, + "p99": 247.33744786935858 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase1_0708_32_4000_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "phase": "phase1", + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 22.435434282000642, + "request_throughput": 5.705260633296099, + "input_token_throughput": 1571.0861468939133, + "output_token_throughput": 753.2726929898756, + "total_token_throughput": 2324.3588398837887, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 5292.305094242806, + "p50": 4425.1284714991925, + "p90": 10562.939984499826, + "p95": 11043.011823458799, + "p99": 11783.398339021078 + }, + "ttft_ms": { + "mean": 336.61362050031585, + "p50": 384.2482510008267, + "p90": 442.5148394977441, + "p95": 448.8336823080317, + "p99": 573.8018302904675 + }, + "tpot_ms": { + "mean": 38.337970145024364, + "p50": 40.33726130479593, + "p90": 49.38140258534856, + "p95": 53.1272594454486, + "p99": 76.12911979381165 + }, + "itl_ms": { + "mean": 37.822721445711366, + "p50": 11.00609399145469, + "p90": 186.46158000046853, + "p95": 188.6525019945111, + "p99": 215.50311118917307 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase1_0708_32_512_256.jsonl" + }, + { + "name": "c1_i131072_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2 + }, + "metrics": { + "success": 2, + "failed": 0, + "duration_s": 9.637961144995643, + "request_throughput": 0.20751276851105258, + "input_token_throughput": 14292.960713120023, + "output_token_throughput": 117.45222697725575, + "total_token_throughput": 14410.41294009728, + "total_input_tokens": 137755, + "total_output_tokens": 1132, + "e2e_ms": { + "mean": 4805.888941002195, + "p50": 4805.888941002195, + "p90": 6773.973971398664, + "p95": 7019.984600198221, + "p99": 7216.793103237869 + }, + "ttft_ms": { + "mean": 465.4355255042901, + "p50": 465.4355255042901, + "p90": 594.7982683050213, + "p95": 610.9686111551127, + "p99": 623.9048854351859 + }, + "tpot_ms": { + "mean": 7.6413752635890955, + "p50": 7.6413752635890955, + "p90": 7.7039553085248675, + "p95": 7.71177781414184, + "p99": 7.718035818635416 + }, + "itl_ms": { + "mean": 7.682129423895364, + "p50": 7.686511991778389, + "p90": 7.886622090882155, + "p95": 7.971059103147127, + "p99": 8.20142485856195 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase2_0708_1_131072_1024.jsonl" + }, + { + "name": "c1_i200000_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 200000, + "output_len": 1024, + "dataset": "random", + "num_prompts": 1 + }, + "metrics": { + "success": 1, + "failed": 0, + "duration_s": 3.999968781005009, + "request_throughput": 0.2500019512024156, + "input_token_throughput": 30489.9879666954, + "output_token_throughput": 109.0008507242532, + "total_token_throughput": 30598.988817419653, + "total_input_tokens": 121959, + "total_output_tokens": 436, + "e2e_ms": { + "mean": 3976.4531809923938, + "p50": 3976.4531809923938, + "p90": 3976.4531809923938, + "p95": 3976.4531809923938, + "p99": 3976.4531809923938 + }, + "ttft_ms": { + "mean": 619.2648659925908, + "p50": 619.2648659925908, + "p90": 619.2648659925908, + "p95": 619.2648659925908, + "p99": 619.2648659925908 + }, + "tpot_ms": { + "mean": 7.7176742873558695, + "p50": 7.7176742873558695, + "p90": 7.7176742873558695, + "p95": 7.7176742873558695, + "p99": 7.7176742873558695 + }, + "itl_ms": { + "mean": 7.717560429859307, + "p50": 7.723745002294891, + "p90": 7.889895391417666, + "p95": 7.947580897598527, + "p99": 8.104919915786013 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase2_0708_1_200000_1024.jsonl" + }, + { + "name": "c1_i32768_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 32768, + "output_len": 1024, + "dataset": "random", + "num_prompts": 8 + }, + "metrics": { + "success": 8, + "failed": 0, + "duration_s": 24.777397109995945, + "request_throughput": 0.3228749155726515, + "input_token_throughput": 4856.523042586037, + "output_token_throughput": 107.75950307237244, + "total_token_throughput": 4964.28254565841, + "total_input_tokens": 120332, + "total_output_tokens": 2670, + "e2e_ms": { + "mean": 3093.2244062478276, + "p50": 3533.7771080012317, + "p90": 4520.390467393736, + "p95": 4747.0502391930495, + "p99": 4928.3780566325 + }, + "ttft_ms": { + "mean": 582.1701634977217, + "p50": 558.6632295016898, + "p90": 967.0584055958898, + "p95": 1088.234538297547, + "p99": 1185.175444458873 + }, + "tpot_ms": { + "mean": 7.551180862107534, + "p50": 7.558533887405872, + "p90": 7.576012124899167, + "p95": 7.579601562424863, + "p99": 7.582473112445421 + }, + "itl_ms": { + "mean": 7.54620997896258, + "p50": 7.535525503044482, + "p90": 7.818251693970524, + "p95": 7.916220599872759, + "p99": 8.15036342246458 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase2_0708_1_32768_1024.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 9.25529791599547, + "request_throughput": 0.4321848995359727, + "input_token_throughput": 12018.954009870517, + "output_token_throughput": 97.34964862047785, + "total_token_throughput": 12116.303658490995, + "total_input_tokens": 111239, + "total_output_tokens": 901, + "e2e_ms": { + "mean": 2307.180852501915, + "p50": 1475.9808770031668, + "p90": 4325.554048095364, + "p95": 4885.47614104318, + "p99": 5333.413815401435 + }, + "ttft_ms": { + "mean": 627.0026992497151, + "p50": 362.3276349972002, + "p90": 1221.7420409055196, + "p95": 1391.1827589574382, + "p99": 1526.7353333989738 + }, + "tpot_ms": { + "mean": 7.564658909780665, + "p50": 7.585828846471562, + "p90": 7.61763043827262, + "p95": 7.621828346514722, + "p99": 7.625186673108404 + }, + "itl_ms": { + "mean": 7.49224082721499, + "p50": 7.479473002604209, + "p90": 7.681275397771969, + "p95": 7.754595801816322, + "p99": 7.96347564610187 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_phase2_0708_1_65536_1024.jsonl" + }, + { + "name": "c1_i32768_o256", + "config": { + "phase": "sharegpt", + "concurrency": 1, + "input_len": 32768, + "output_len": 256, + "dataset": "sharegpt", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 66.9215779669903, + "request_throughput": 0.4781716297213479, + "input_token_throughput": 148.7113768433392, + "output_token_throughput": 122.41193720866507, + "total_token_throughput": 271.1233140520043, + "total_input_tokens": 9952, + "total_output_tokens": 8192, + "e2e_ms": { + "mean": 2089.52630365593, + "p50": 2085.8885635025217, + "p90": 2120.287249701505, + "p95": 2121.837336999306, + "p99": 2126.285889654391 + }, + "ttft_ms": { + "mean": 194.07644296870785, + "p50": 188.67110900464468, + "p90": 223.09383450337918, + "p95": 225.2502156974515, + "p99": 228.79413440998178 + }, + "tpot_ms": { + "mean": 7.433136708577342, + "p50": 7.432712123509012, + "p90": 7.441393502750525, + "p95": 7.442859282766186, + "p99": 7.444249214380797 + }, + "itl_ms": { + "mean": 7.433842794214755, + "p50": 7.431341000483371, + "p90": 7.5934758060611784, + "p95": 7.654351103701629, + "p99": 7.860458772629499 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_sharegpt_0708_1_32768_256.jsonl" + }, + { + "name": "c32_i32768_o256", + "config": { + "phase": "sharegpt", + "concurrency": 32, + "input_len": 32768, + "output_len": 256, + "dataset": "sharegpt", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 14.057225968994317, + "request_throughput": 9.105637220481944, + "input_token_throughput": 3273.0497540185484, + "output_token_throughput": 2331.0431284433776, + "total_token_throughput": 5604.092882461926, + "total_input_tokens": 46010, + "total_output_tokens": 32768, + "e2e_ms": { + "mean": 3490.736479040038, + "p50": 3548.8332320019254, + "p90": 3701.91772600665, + "p95": 3703.1692250500782, + "p99": 3706.4754708172404 + }, + "ttft_ms": { + "mean": 629.6461200562362, + "p50": 699.2854104973958, + "p90": 809.4456622115104, + "p95": 894.5322211977327, + "p99": 897.1345172059955 + }, + "tpot_ms": { + "mean": 11.21996219209334, + "p50": 10.98478311568946, + "p90": 11.782473441545495, + "p95": 11.79308803883032, + "p99": 12.381156532274785 + }, + "itl_ms": { + "mean": 11.221238664603872, + "p50": 10.92621900170343, + "p90": 11.17239399900427, + "p95": 11.247147751419106, + "p99": 12.367407945566937 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/raw_outputs/sglang_sharegpt_0708_32_32768_256.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/report.md b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/report.md new file mode 100644 index 0000000..a7b672d --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/report.md @@ -0,0 +1,25 @@ +# H200 VLLM Comparison Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: VLLM (TP=8) +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | phase1 | 128 | 512 | 256 | 70.29 | 128 | 1.82 | 501.46 | 240.43 | 741.88 | 34320.42 | 65775.64 | 68346.91 | 7.51 | 8.88 | 9.95 | 35303.54 | 66450.82 | 69371.23 | ⚠️ | +| c1_i4000_o512 | phase1 | 1 | 4000 | 512 | 64.85 | 32 | 0.49 | 1002.32 | 138.92 | 1141.24 | 147.94 | 195.78 | 198.56 | 6.68 | 6.73 | 6.73 | 2025.72 | 3528.12 | 3604.09 | ✅ | +| c1_i512_o256 | phase1 | 1 | 512 | 256 | 34.95 | 32 | 0.92 | 233.76 | 133.24 | 367.00 | 128.49 | 188.08 | 196.56 | 6.67 | 6.71 | 6.73 | 1091.23 | 1847.17 | 1877.61 | ✅ | +| c32_i4000_o512 | phase1 | 32 | 4000 | 512 | 68.59 | 64 | 0.93 | 1959.81 | 263.43 | 2223.24 | 23950.97 | 33396.12 | 33949.08 | 7.20 | 7.61 | 8.33 | 25978.04 | 35750.60 | 36115.49 | ⚠️ | +| c32_i512_o256 | phase1 | 32 | 512 | 256 | 70.63 | 128 | 1.81 | 499.03 | 239.27 | 738.30 | 14403.12 | 17481.79 | 18082.76 | 7.54 | 9.09 | 9.97 | 15389.16 | 18637.78 | 19259.08 | ⚠️ | +| c1_i131072_o1024 | phase2 | 1 | 131072 | 1024 | 8.45 | 2 | 0.24 | 16305.61 | 133.99 | 16439.60 | 369.70 | 549.11 | 565.06 | 6.79 | 6.84 | 6.84 | 4222.08 | 6232.30 | 6410.99 | ✅ | +| c1_i200000_o1024 | phase2 | 1 | 200000 | 1024 | 3.59 | 1 | 0.28 | 33993.26 | 121.52 | 34114.79 | 608.85 | 608.85 | 608.85 | 6.84 | 6.84 | 6.84 | 3584.28 | 3584.28 | 3584.28 | ✅ | +| c1_i32768_o1024 | phase2 | 1 | 32768 | 1024 | 22.20 | 8 | 0.36 | 5420.68 | 120.28 | 5540.95 | 537.39 | 1007.75 | 1099.08 | 6.70 | 6.76 | 6.76 | 2773.61 | 4240.33 | 4391.50 | ✅ | +| c1_i65536_o1024 | phase2 | 1 | 65536 | 1024 | 7.97 | 4 | 0.50 | 13956.46 | 113.04 | 14069.51 | 512.79 | 1199.14 | 1320.28 | 6.11 | 6.66 | 6.67 | 1990.96 | 4285.04 | 4685.90 | ✅ | +| c1_i32768_o256 | sharegpt | 1 | 32768 | 256 | 60.52 | 32 | 0.53 | 164.44 | 135.36 | 299.80 | 192.81 | 319.19 | 332.05 | 6.66 | 6.66 | 6.67 | 1890.26 | 2015.91 | 2026.78 | ✅ | +| c32_i32768_o256 | sharegpt | 32 | 32768 | 256 | 124.49 | 128 | 1.03 | 369.58 | 263.21 | 632.79 | 25671.36 | 29346.73 | 29758.65 | 7.18 | 7.46 | 7.48 | 27502.28 | 31173.56 | 31587.45 | ⚠️ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/results.json b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/results.json new file mode 100644 index 0000000..aecd84c --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/results.json @@ -0,0 +1,646 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang_vs_vllm_vllm", + "run_id": "20260708-083346", + "timestamp": "2026-07-08T08:44:53+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "0135a5f", + "git_dirty": "dirty", + "description": "H200 vllm TP=8 comparison benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_sglang_vs_vllm/start_vllm.sh", + "phase1_server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30005", + "phase2_server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 210000 --max-num-seqs 2 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30005" + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "phase": "phase1", + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 70.29131372000847, + "request_throughput": 1.8209931387804565, + "input_token_throughput": 501.4559855916682, + "output_token_throughput": 240.42800035460715, + "total_token_throughput": 741.8839859462754, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 35303.543292343646, + "p50": 35317.00481100415, + "p90": 62426.622139496605, + "p95": 66450.8159798068, + "p99": 69371.2297050588 + }, + "ttft_ms": { + "mean": 34320.41640907778, + "p50": 34523.768147002556, + "p90": 61465.390575592755, + "p95": 65775.64489539945, + "p99": 68346.90796818773 + }, + "tpot_ms": { + "mean": 7.510403795740832, + "p50": 7.2786697395865, + "p90": 8.519100261819153, + "p95": 8.883510504525162, + "p99": 9.947404425221317 + }, + "itl_ms": { + "mean": 7.5049212809090236, + "p50": 6.796126006520353, + "p90": 6.991922602173872, + "p95": 7.084895302250516, + "p99": 8.646592354052709 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase1_0708_128_512_256.jsonl" + }, + { + "name": "c1_i4000_o512", + "config": { + "phase": "phase1", + "concurrency": 1, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 64.85143358699861, + "request_throughput": 0.49343550681993475, + "input_token_throughput": 1002.3217129471687, + "output_token_throughput": 138.91751502939977, + "total_token_throughput": 1141.2392279765686, + "total_input_tokens": 65002, + "total_output_tokens": 9009, + "e2e_ms": { + "mean": 2025.7195414683338, + "p50": 2064.7940229973756, + "p90": 3377.0389970930414, + "p95": 3528.1217303978337, + "p99": 3604.093828440673 + }, + "ttft_ms": { + "mean": 147.9420001883227, + "p50": 142.5741455022944, + "p90": 185.8226598065812, + "p95": 195.784989852109, + "p99": 198.56474577492918 + }, + "tpot_ms": { + "mean": 6.682914310707279, + "p50": 6.6611079833776214, + "p90": 6.731920153351648, + "p95": 6.73228240863953, + "p99": 6.732959319151181 + }, + "itl_ms": { + "mean": 6.693498746242357, + "p50": 6.696365991956554, + "p90": 6.864056596532464, + "p95": 6.913123605772852, + "p99": 7.0496275601908565 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase1_0708_1_4000_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "phase": "phase1", + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 34.95065580600931, + "request_throughput": 0.915576525305085, + "input_token_throughput": 233.75813161695453, + "output_token_throughput": 133.24499619830567, + "total_token_throughput": 367.0031278152602, + "total_input_tokens": 8170, + "total_output_tokens": 4657, + "e2e_ms": { + "mean": 1091.2277118441125, + "p50": 1291.4932084968314, + "p90": 1796.9704142975388, + "p95": 1847.1714119055832, + "p99": 1877.6105558480776 + }, + "ttft_ms": { + "mean": 128.49185015784315, + "p50": 112.28471150388941, + "p90": 186.1637935100589, + "p95": 188.0846640546224, + "p99": 196.56408056471264 + }, + "tpot_ms": { + "mean": 6.669301239421435, + "p50": 6.6644271957983365, + "p90": 6.685313248133753, + "p95": 6.706201844159902, + "p99": 6.732251316311054 + }, + "itl_ms": { + "mean": 6.660766289726756, + "p50": 6.646345995250158, + "p90": 6.851249004830606, + "p95": 6.929318403126672, + "p99": 7.189810206182302 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase1_0708_1_512_256.jsonl" + }, + { + "name": "c32_i4000_o512", + "config": { + "phase": "phase1", + "concurrency": 32, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 64 + }, + "metrics": { + "success": 64, + "failed": 0, + "duration_s": 68.59192536699993, + "request_throughput": 0.9330544325380734, + "input_token_throughput": 1959.807940668681, + "output_token_throughput": 263.42750846141325, + "total_token_throughput": 2223.2354491300944, + "total_input_tokens": 134427, + "total_output_tokens": 18069, + "e2e_ms": { + "mean": 25978.04190787588, + "p50": 32999.21867699595, + "p90": 35292.39189830696, + "p95": 35750.602176762186, + "p99": 36115.4935479541 + }, + "ttft_ms": { + "mean": 23950.969101952294, + "p50": 31194.911923492327, + "p90": 33219.88859529374, + "p95": 33396.116484549566, + "p99": 33949.08106612915 + }, + "tpot_ms": { + "mean": 7.195712811761743, + "p50": 7.178219830203397, + "p90": 7.427465439651535, + "p95": 7.608804859536966, + "p99": 8.330944953500406 + }, + "itl_ms": { + "mean": 7.205215613496251, + "p50": 6.882653993670829, + "p90": 7.040518394205719, + "p95": 7.092614797875285, + "p99": 7.720463996520265 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase1_0708_32_4000_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "phase": "phase1", + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 70.6325325450016, + "request_throughput": 1.8121960998417872, + "input_token_throughput": 499.0335009939322, + "output_token_throughput": 239.26651630723597, + "total_token_throughput": 738.3000173011682, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 15389.164376773351, + "p50": 17040.255194995552, + "p90": 18396.107898907212, + "p95": 18637.781482394348, + "p99": 19259.082938716165 + }, + "ttft_ms": { + "mean": 14403.11916330495, + "p50": 16087.491760001285, + "p90": 17194.928123198042, + "p95": 17481.788155655522, + "p99": 18082.755415009597 + }, + "tpot_ms": { + "mean": 7.540611207711906, + "p50": 7.276942960920789, + "p90": 8.697657064072803, + "p95": 9.088215577041193, + "p99": 9.965460682700025 + }, + "itl_ms": { + "mean": 7.52855506722901, + "p50": 6.799451999540906, + "p90": 6.994641700293869, + "p95": 7.1075246523832885, + "p99": 8.330840613489272 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase1_0708_32_512_256.jsonl" + }, + { + "name": "c1_i131072_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2 + }, + "metrics": { + "success": 2, + "failed": 0, + "duration_s": 8.44832066699746, + "request_throughput": 0.23673343837584251, + "input_token_throughput": 16305.607401732092, + "output_token_throughput": 133.99112612072685, + "total_token_throughput": 16439.59852785282, + "total_input_tokens": 137755, + "total_output_tokens": 1132, + "e2e_ms": { + "mean": 4222.08211600082, + "p50": 4222.08211600082, + "p90": 6008.944166405126, + "p95": 6232.301922705665, + "p99": 6410.988127746095 + }, + "ttft_ms": { + "mean": 369.69626199424965, + "p50": 369.69626199424965, + "p90": 529.1791395968175, + "p95": 549.1144992971385, + "p99": 565.0627870573953 + }, + "tpot_ms": { + "mean": 6.789414029039928, + "p50": 6.789414029039928, + "p90": 6.833798917444026, + "p95": 6.839347028494538, + "p99": 6.843785517334948 + }, + "itl_ms": { + "mean": 6.891481741492917, + "p50": 6.91215450206073, + "p90": 7.0499842942808755, + "p95": 7.094744750065729, + "p99": 7.19807219240465 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase2_0708_1_131072_1024.jsonl" + }, + { + "name": "c1_i200000_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 200000, + "output_len": 1024, + "dataset": "random", + "num_prompts": 1 + }, + "metrics": { + "success": 1, + "failed": 0, + "duration_s": 3.587740169998142, + "request_throughput": 0.2787269848475448, + "input_token_throughput": 33993.26434502172, + "output_token_throughput": 121.52496539352954, + "total_token_throughput": 34114.789310415246, + "total_input_tokens": 121959, + "total_output_tokens": 436, + "e2e_ms": { + "mean": 3584.2828049935633, + "p50": 3584.2828049935633, + "p90": 3584.2828049935633, + "p95": 3584.2828049935633, + "p99": 3584.2828049935633 + }, + "ttft_ms": { + "mean": 608.8514690054581, + "p50": 608.8514690054581, + "p90": 608.8514690054581, + "p95": 608.8514690054581, + "p99": 608.8514690054581 + }, + "tpot_ms": { + "mean": 6.840072036754266, + "p50": 6.840072036754266, + "p90": 6.840072036754266, + "p95": 6.840072036754266, + "p99": 6.840072036754266 + }, + "itl_ms": { + "mean": 6.93562900699623, + "p50": 6.93584700638894, + "p90": 7.094994801445864, + "p95": 7.155361198238097, + "p99": 7.659021359286264 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase2_0708_1_200000_1024.jsonl" + }, + { + "name": "c1_i32768_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 32768, + "output_len": 1024, + "dataset": "random", + "num_prompts": 8 + }, + "metrics": { + "success": 8, + "failed": 0, + "duration_s": 22.198709311996936, + "request_throughput": 0.36038131260525713, + "input_token_throughput": 5420.675513551975, + "output_token_throughput": 120.27726308200457, + "total_token_throughput": 5540.95277663398, + "total_input_tokens": 120332, + "total_output_tokens": 2670, + "e2e_ms": { + "mean": 2773.605513126313, + "p50": 3165.8916160013177, + "p90": 4051.3738169975113, + "p95": 4240.332765497442, + "p99": 4391.499924297385 + }, + "ttft_ms": { + "mean": 537.3895926240948, + "p50": 515.5716105000465, + "p90": 893.5937416943488, + "p95": 1007.7544418440083, + "p99": 1099.0830019637358 + }, + "tpot_ms": { + "mean": 6.695183012876431, + "p50": 6.722612965778395, + "p90": 6.757049449913822, + "p95": 6.759753052316733, + "p99": 6.761915934239061 + }, + "itl_ms": { + "mean": 6.748137609582467, + "p50": 6.770501000573859, + "p90": 6.986180000239983, + "p95": 7.086728001013398, + "p99": 7.310383494768757 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase2_0708_1_32768_1024.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "phase2", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 7.970429492008407, + "request_throughput": 0.5018550134607703, + "input_token_throughput": 13956.46246059066, + "output_token_throughput": 113.04284178203852, + "total_token_throughput": 14069.505302372698, + "total_input_tokens": 111239, + "total_output_tokens": 901, + "e2e_ms": { + "mean": 1990.9638142526092, + "p50": 1233.321038504073, + "p90": 3783.9562135966853, + "p95": 4285.0388272949185, + "p99": 4685.904918253508 + }, + "ttft_ms": { + "mean": 512.7911880008469, + "p50": 292.3798990013893, + "p90": 1047.7265454042936, + "p95": 1199.1442872065813, + "p99": 1320.278480648412 + }, + "tpot_ms": { + "mean": 6.105170133133754, + "p50": 6.499210690651045, + "p90": 6.645387070930873, + "p95": 6.658606761180735, + "p99": 6.669182513380624 + }, + "itl_ms": { + "mean": 6.680791734456016, + "p50": 6.692053997539915, + "p90": 6.890294799814001, + "p95": 6.961315794615075, + "p99": 7.188904156791978 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_phase2_0708_1_65536_1024.jsonl" + }, + { + "name": "c1_i32768_o256", + "config": { + "phase": "sharegpt", + "concurrency": 1, + "input_len": 32768, + "output_len": 256, + "dataset": "sharegpt", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 60.52097901000525, + "request_throughput": 0.5287422729019271, + "input_token_throughput": 164.43884687249934, + "output_token_throughput": 135.35802186289334, + "total_token_throughput": 299.7968687353927, + "total_input_tokens": 9952, + "total_output_tokens": 8192, + "e2e_ms": { + "mean": 1890.2617764370007, + "p50": 1880.618217990559, + "p90": 2011.3904219004323, + "p95": 2015.9103012978449, + "p99": 2026.7773708580355 + }, + "ttft_ms": { + "mean": 192.80972293654486, + "p50": 182.4396789961611, + "p90": 313.661693801987, + "p95": 319.1885438936879, + "p99": 332.0496352396731 + }, + "tpot_ms": { + "mean": 6.65667471960963, + "p50": 6.656709717622018, + "p90": 6.663090442341534, + "p95": 6.6648275378375885, + "p99": 6.6690332513535395 + }, + "itl_ms": { + "mean": 6.658109007481169, + "p50": 6.642145999649074, + "p90": 6.822339407517575, + "p95": 6.881276096828515, + "p99": 7.053025057684864 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_sharegpt_0708_1_32768_256.jsonl" + }, + { + "name": "c32_i32768_o256", + "config": { + "phase": "sharegpt", + "concurrency": 32, + "input_len": 32768, + "output_len": 256, + "dataset": "sharegpt", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 124.49392829900899, + "request_throughput": 1.0281625919343644, + "input_token_throughput": 369.5762566789071, + "output_token_throughput": 263.2096235351973, + "total_token_throughput": 632.7858802141044, + "total_input_tokens": 46010, + "total_output_tokens": 32768, + "e2e_ms": { + "mean": 27502.27682496768, + "p50": 30894.017616497877, + "p90": 31096.156101305678, + "p95": 31173.55504575462, + "p99": 31587.450084360753 + }, + "ttft_ms": { + "mean": 25671.35998733522, + "p50": 29065.171126996574, + "p90": 29269.512161599414, + "p95": 29346.73411935073, + "p99": 29758.65151028571 + }, + "tpot_ms": { + "mean": 7.180066029931208, + "p50": 7.160704684307953, + "p90": 7.288568628234012, + "p95": 7.456718741379126, + "p99": 7.484840608305212 + }, + "itl_ms": { + "mean": 7.191130708828689, + "p50": 6.7904099996667355, + "p90": 6.969844806008041, + "p95": 7.025835799868219, + "p99": 7.566049310844388 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/raw_outputs/vllm_sharegpt_0708_32_32768_256.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh index e69764f..56d738d 100755 --- a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh +++ b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh @@ -308,27 +308,19 @@ write_backend_metadata() { "H200 ${backend} TP=8 comparison benchmark for DeepSeek-V4-Flash" # Embed config and server args. - "${VENV_CLIENT}/bin/python" - "$meta_json" "$backend" "$phase1_args" "$phase2_args" <<'PY' -import json -import sys - -path, backend, phase1_args, phase2_args = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 8, - "cuda_visible_devices": "0,1,2,3,4,5,6,7", - "phase1_max_model_len": 32768, - "phase2_max_model_len": 210000, - "backend": backend, - "server_start_script": f"experiments/dsv4_h200_sglang_vs_vllm/start_{backend}.sh", - "phase1_server_args": phase1_args, - "phase2_server_args": phase2_args, -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY + jq --arg backend "$backend" \ + --arg phase1_args "$phase1_args" \ + --arg phase2_args "$phase2_args" \ + '.config = { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": $backend, + "server_start_script": "experiments/dsv4_h200_sglang_vs_vllm/start_\($backend).sh", + "phase1_server_args": $phase1_args, + "phase2_server_args": $phase2_args + }' "$meta_json" > "${meta_json}.tmp" && mv "${meta_json}.tmp" "$meta_json" } # ---------------------------------------------------------------------------