diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/comparison.md b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/comparison.md new file mode 100644 index 0000000..707388d --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/comparison.md @@ -0,0 +1,30 @@ +# vLLM+DSpark vs vLLM default on DeepSeek-V4-Flash (H200, TP=8) + +## Summary + +- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark` +- Default model: `/data/models/DeepSeek-V4-Flash` +- Hardware: 8x NVIDIA H200 143GB +- Tensor Parallelism: 8 +- Benchmark client: `sglang.bench_serving --backend vllm` +- DSpark flags: `--spec-method dspark --spec-model --spec-tokens 5` + +## Side-by-side results (TTFT focused) + +| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | vllm-dspark | 128 | 512 | 256 | 23.94 | 3160.81 | 778.36 | 1071.85 | 1075.35 | 28.95 | 78.75 | 3632.34 | +| c128_i512_o256 | vllm-default | 128 | 512 | 256 | 29.99 | 3959.89 | 617.72 | 767.18 | 770.89 | 18.31 | 65.44 | 2695.91 | +| c1_i4000_o512 | vllm-dspark | 1 | 4000 | 512 | 1.73 | 486.20 | 123.58 | 176.29 | 190.52 | 1.66 | 3.56 | 578.26 | +| c1_i4000_o512 | vllm-default | 1 | 4000 | 512 | 0.50 | 140.13 | 131.50 | 191.53 | 268.69 | 6.68 | 6.73 | 2008.09 | +| c1_i512_o256 | vllm-dspark | 1 | 512 | 256 | 2.56 | 372.62 | 86.51 | 185.00 | 188.81 | 2.09 | 3.83 | 389.62 | +| c1_i512_o256 | vllm-default | 1 | 512 | 256 | 0.99 | 143.82 | 49.50 | 68.50 | 71.09 | 6.66 | 6.71 | 1010.98 | +| c32_i4000_o512 | vllm-dspark | 32 | 4000 | 512 | 11.49 | 3242.88 | 298.42 | 434.24 | 522.34 | 9.68 | 54.60 | 2476.82 | +| c32_i4000_o512 | vllm-default | 32 | 4000 | 512 | 5.38 | 1518.49 | 382.74 | 681.98 | 684.75 | 16.43 | 25.49 | 4927.71 | +| c32_i512_o256 | vllm-dspark | 32 | 512 | 256 | 16.53 | 2182.83 | 262.56 | 527.21 | 532.11 | 12.06 | 28.16 | 1738.79 | +| c32_i512_o256 | vllm-default | 32 | 512 | 256 | 12.98 | 1713.20 | 122.62 | 272.00 | 304.80 | 15.77 | 23.26 | 2154.65 | + +## Notes + +- TTFT mean/P95/P99 are the main focus for verifying whether DSpark increases time-to-first-token. +- Mean TPOT and E2E are included to check whether speculative decoding pays back after first token. diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/report.md b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/report.md new file mode 100644 index 0000000..90f024f --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/report.md @@ -0,0 +1,18 @@ +# H200 vLLM default Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default` +- Backend: vLLM default (TP=8) +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | 128 | 512 | 256 | 4.27 | 128 | 29.99 | 8259.06 | 3959.89 | 12218.95 | 617.72 | 767.18 | 770.89 | 18.31 | 28.68 | 65.44 | 2695.91 | 4153.36 | 4205.54 | ✅ | +| c1_i4000_o512 | 1 | 4000 | 512 | 64.29 | 32 | 0.50 | 1011.09 | 140.13 | 1151.23 | 131.50 | 191.53 | 268.69 | 6.68 | 6.73 | 6.73 | 2008.09 | 3523.51 | 3599.32 | ✅ | +| c1_i512_o256 | 1 | 512 | 256 | 32.38 | 32 | 0.99 | 252.30 | 143.82 | 396.12 | 49.50 | 68.50 | 71.09 | 6.66 | 6.69 | 6.71 | 1010.98 | 1735.31 | 1759.78 | ✅ | +| c32_i4000_o512 | 32 | 4000 | 512 | 11.90 | 64 | 5.38 | 11297.00 | 1518.49 | 12815.49 | 382.74 | 681.98 | 684.75 | 16.43 | 20.90 | 25.49 | 4927.71 | 8743.37 | 9349.37 | ✅ | +| c32_i512_o256 | 32 | 512 | 256 | 9.86 | 128 | 12.98 | 3573.19 | 1713.20 | 5286.40 | 122.62 | 272.00 | 304.80 | 15.77 | 19.50 | 23.26 | 2154.65 | 4102.71 | 4275.05 | ✅ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/results.json b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/results.json new file mode 100644 index 0000000..44fc6e0 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/results.json @@ -0,0 +1,304 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_dspark_vs_default_default", + "run_id": "20260708-142121", + "timestamp": "2026-07-08T14:21:25+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm-default", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "1a6b173", + "git_dirty": "dirty", + "description": "H200 vllm-default TP=8 benchmark for TTFT comparison" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "max_model_len": 32768, + "max_num_seqs": 256, + "backend": "default", + "server_start_script": "experiments/dsv4_h200_vllm_dspark_vs_default/start_default.sh", + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30004" + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 4.267798511005822, + "request_throughput": 29.992043830071385, + "input_token_throughput": 8259.059069705907, + "output_token_throughput": 3959.8870369391125, + "total_token_throughput": 12218.94610664502, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 2695.9114590795252, + "p50": 2786.5102920040954, + "p90": 4094.3853520046105, + "p95": 4153.361671900348, + "p99": 4205.538639774604 + }, + "ttft_ms": { + "mean": 617.7214959924413, + "p50": 563.3532095089322, + "p90": 764.0312482006266, + "p95": 767.1767810017627, + "p99": 770.8853784023086 + }, + "tpot_ms": { + "mean": 18.314879062516017, + "p50": 15.941864871586276, + "p90": 23.19956007269117, + "p95": 28.68311282203317, + "p99": 65.43692430894995 + }, + "itl_ms": { + "mean": 15.86385086378528, + "p50": 14.933839993318543, + "p90": 17.340479693666566, + "p95": 18.231574493984226, + "p99": 21.184932928154012 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_128_512_256.jsonl" + }, + { + "name": "c1_i4000_o512", + "config": { + "concurrency": 1, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 64.28883961599786, + "request_throughput": 0.49775357886591887, + "input_token_throughput": 1011.0930666700768, + "output_token_throughput": 140.13318725009572, + "total_token_throughput": 1151.2262539201727, + "total_input_tokens": 65002, + "total_output_tokens": 9009, + "e2e_ms": { + "mean": 2008.0858264368544, + "p50": 2064.2024725020747, + "p90": 3374.621986798593, + "p95": 3523.513536003884, + "p99": 3599.319600486051 + }, + "ttft_ms": { + "mean": 131.49705237538, + "p50": 128.75337050354574, + "p90": 188.4824504086282, + "p95": 191.5273676990182, + "p99": 268.6876055170434 + }, + "tpot_ms": { + "mean": 6.678438385450706, + "p50": 6.647801321133651, + "p90": 6.729117710662969, + "p95": 6.730015678053446, + "p99": 6.730549147956696 + }, + "itl_ms": { + "mean": 6.689239366715674, + "p50": 6.692727998597547, + "p90": 6.8432597909122705, + "p95": 6.884838393307291, + "p99": 6.988287522690371 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_1_4000_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 32.381663249005214, + "request_throughput": 0.9882135995896709, + "input_token_throughput": 252.30328464523785, + "output_token_throughput": 143.81596041528428, + "total_token_throughput": 396.11924506052213, + "total_input_tokens": 8170, + "total_output_tokens": 4657, + "e2e_ms": { + "mean": 1010.982028125909, + "p50": 1195.345941996493, + "p90": 1688.3191696979338, + "p95": 1735.3133527984028, + "p99": 1759.781412960583 + }, + "ttft_ms": { + "mean": 49.50495168895941, + "p50": 44.729964502039365, + "p90": 65.38926539797103, + "p95": 68.496801002766, + "p99": 71.08588116068859 + }, + "tpot_ms": { + "mean": 6.659676006644546, + "p50": 6.654429546349644, + "p90": 6.684645557662856, + "p95": 6.69398312746385, + "p99": 6.711085371917718 + }, + "itl_ms": { + "mean": 6.652042307463047, + "p50": 6.637780999881215, + "p90": 6.787886397796684, + "p95": 6.841546000214294, + "p99": 7.143319878960029 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_1_512_256.jsonl" + }, + { + "name": "c32_i4000_o512", + "config": { + "concurrency": 32, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 64 + }, + "metrics": { + "success": 64, + "failed": 0, + "duration_s": 11.899348942009965, + "request_throughput": 5.378445519321792, + "input_token_throughput": 11297.004622279226, + "output_token_throughput": 1518.4864388847727, + "total_token_throughput": 12815.491061163999, + "total_input_tokens": 134427, + "total_output_tokens": 18069, + "e2e_ms": { + "mean": 4927.714462639642, + "p50": 4734.132497498649, + "p90": 8425.151473002916, + "p95": 8743.374133249745, + "p99": 9349.369201347727 + }, + "ttft_ms": { + "mean": 382.74424496808024, + "p50": 390.3064074984286, + "p90": 678.5326104989508, + "p95": 681.9790261564776, + "p99": 684.7464643983403 + }, + "tpot_ms": { + "mean": 16.425838503257715, + "p50": 17.044603306082735, + "p90": 18.812656380989317, + "p95": 20.897049261926014, + "p99": 25.487691384272917 + }, + "itl_ms": { + "mean": 16.239184021720266, + "p50": 11.25292000506306, + "p90": 12.360760505544022, + "p95": 48.09924705259619, + "p99": 161.26204263797263 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_32_4000_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 9.864565095995204, + "request_throughput": 12.975736766333995, + "input_token_throughput": 3573.1935120292237, + "output_token_throughput": 1713.2027449300354, + "total_token_throughput": 5286.3962569592595, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 2154.6506077650065, + "p50": 2170.227252499899, + "p90": 3869.9233716019075, + "p95": 4102.707951706542, + "p99": 4275.054011319589 + }, + "ttft_ms": { + "mean": 122.61577058620787, + "p50": 76.25793749321019, + "p90": 266.052756505087, + "p95": 272.00344715092797, + "p99": 304.79746079756296 + }, + "tpot_ms": { + "mean": 15.771494928385172, + "p50": 16.114622466954064, + "p90": 17.704943566836604, + "p95": 19.49543390091434, + "p99": 23.258708736311746 + }, + "itl_ms": { + "mean": 15.513364437561801, + "p50": 11.447760996816214, + "p90": 26.607878498907667, + "p95": 30.321172751428094, + "p99": 100.95071544419625 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_32_512_256.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/report.md b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/report.md new file mode 100644 index 0000000..a369c54 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/report.md @@ -0,0 +1,18 @@ +# H200 vLLM+DSpark Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark` +- Backend: vLLM+DSpark (TP=8) +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | 128 | 512 | 256 | 5.35 | 128 | 23.94 | 6592.44 | 3160.81 | 9753.24 | 778.36 | 1071.85 | 1075.35 | 28.95 | 69.00 | 78.75 | 3632.34 | 5022.66 | 5229.14 | ✅ | +| c1_i4000_o512 | 1 | 4000 | 512 | 18.53 | 32 | 1.73 | 3508.05 | 486.20 | 3994.25 | 123.58 | 176.29 | 190.52 | 1.66 | 2.30 | 3.56 | 578.26 | 964.23 | 1147.23 | ✅ | +| c1_i512_o256 | 1 | 512 | 256 | 12.50 | 32 | 2.56 | 653.70 | 372.62 | 1026.32 | 86.51 | 185.00 | 188.81 | 2.09 | 3.41 | 3.83 | 389.62 | 777.58 | 915.24 | ✅ | +| c32_i4000_o512 | 32 | 4000 | 512 | 5.57 | 64 | 11.49 | 24125.91 | 3242.88 | 27368.79 | 298.42 | 434.24 | 522.34 | 9.68 | 14.02 | 54.60 | 2476.82 | 4615.41 | 4710.26 | ✅ | +| c32_i512_o256 | 32 | 512 | 256 | 7.74 | 128 | 16.53 | 4552.68 | 2182.83 | 6735.51 | 262.56 | 527.21 | 532.11 | 12.06 | 22.73 | 28.16 | 1738.79 | 3830.80 | 5143.02 | ✅ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/results.json b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/results.json new file mode 100644 index 0000000..7e03144 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/results.json @@ -0,0 +1,304 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_dspark_vs_default_dspark", + "run_id": "20260708-142121", + "timestamp": "2026-07-08T14:21:25+00:00", + "model": "/data/models/DeepSeek-V4-Flash-DSpark", + "backend": "vllm", + "engine": "vllm-dspark", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh", + "env": "/data/user1/yy/envs/vllm-dspark", + "git_commit": "1a6b173", + "git_dirty": "dirty", + "description": "H200 vllm-dspark TP=8 benchmark for TTFT comparison" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "max_model_len": 32768, + "max_num_seqs": 256, + "backend": "dspark", + "server_start_script": "experiments/dsv4_h200_vllm_dspark_vs_default/start_dspark.sh", + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash-DSpark --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --spec-method dspark --spec-model /data/models/DeepSeek-V4-Flash-DSpark --spec-tokens 5 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30007" + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 5.346734229999129, + "request_throughput": 23.939847109255112, + "input_token_throughput": 6592.435397711127, + "output_token_throughput": 3160.807938643839, + "total_token_throughput": 9753.243336354966, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 3632.342373304141, + "p50": 4026.272206501744, + "p90": 4568.2280586028355, + "p95": 5022.660237102536, + "p99": 5229.135414937482 + }, + "ttft_ms": { + "mean": 778.3609630697583, + "p50": 860.7529874934698, + "p90": 1069.5359163975809, + "p95": 1071.847062408051, + "p99": 1075.3475757954584 + }, + "tpot_ms": { + "mean": 28.950386059739248, + "p50": 25.412556098403837, + "p90": 47.2479180524213, + "p95": 69.00331620594505, + "p99": 78.74700601221775 + }, + "itl_ms": { + "mean": 93.81264048253927, + "p50": 40.38110149849672, + "p90": 167.10851929528872, + "p95": 178.33137240304495, + "p99": 226.998560061038 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_128_512_256.jsonl" + }, + { + "name": "c1_i4000_o512", + "config": { + "concurrency": 1, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 18.529377975006355, + "request_throughput": 1.7269872762681893, + "input_token_throughput": 3508.050841624526, + "output_token_throughput": 486.2008866218786, + "total_token_throughput": 3994.2517282464046, + "total_input_tokens": 65002, + "total_output_tokens": 9009, + "e2e_ms": { + "mean": 578.2551125621467, + "p50": 591.3451755041024, + "p90": 892.7950516954298, + "p95": 964.232088495919, + "p99": 1147.2280103387313 + }, + "ttft_ms": { + "mean": 123.576120467078, + "p50": 116.67309649055824, + "p90": 165.92405038827567, + "p95": 176.29006390707215, + "p99": 190.51729560960666 + }, + "tpot_ms": { + "mean": 1.663761482756609, + "p50": 1.5393092801409114, + "p90": 1.721910965739694, + "p95": 2.2984976072946695, + "p99": 3.557179712464779 + }, + "itl_ms": { + "mean": 8.58824287840908, + "p50": 8.61248900037026, + "p90": 8.832874392101076, + "p95": 8.92088470282033, + "p99": 9.17154152484727 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_1_4000_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 12.498076534990105, + "request_throughput": 2.560393986259529, + "input_token_throughput": 653.7005896168861, + "output_token_throughput": 372.61733731283215, + "total_token_throughput": 1026.3179269297182, + "total_input_tokens": 8170, + "total_output_tokens": 4657, + "e2e_ms": { + "mean": 389.62139918930916, + "p50": 377.8842995016021, + "p90": 646.4735688015936, + "p95": 777.5833995023276, + "p99": 915.2371472108645 + }, + "ttft_ms": { + "mean": 86.50895553273585, + "p50": 65.89411200548057, + "p90": 180.50938470114488, + "p95": 184.9997164463275, + "p99": 188.8067441045132 + }, + "tpot_ms": { + "mean": 2.0946395465617687, + "p50": 1.810156284521167, + "p90": 3.2686956743572244, + "p95": 3.410268214368921, + "p99": 3.8341900713573533 + }, + "itl_ms": { + "mean": 8.582383006246063, + "p50": 8.498353497998323, + "p90": 8.866577497974504, + "p95": 9.046812405722449, + "p99": 11.975977418769617 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_1_512_256.jsonl" + }, + { + "name": "c32_i4000_o512", + "config": { + "concurrency": 32, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 64 + }, + "metrics": { + "success": 64, + "failed": 0, + "duration_s": 5.57189327001106, + "request_throughput": 11.486221450877318, + "input_token_throughput": 24125.910796516957, + "output_token_throughput": 3242.883365560973, + "total_token_throughput": 27368.794162077927, + "total_input_tokens": 134427, + "total_output_tokens": 18069, + "e2e_ms": { + "mean": 2476.816028921121, + "p50": 2355.44883399416, + "p90": 4230.580038897461, + "p95": 4615.411171296, + "p99": 4710.255208882008 + }, + "ttft_ms": { + "mean": 298.4201381239018, + "p50": 329.9452685023425, + "p90": 424.3153512070421, + "p95": 434.23709184717154, + "p99": 522.339181041898 + }, + "tpot_ms": { + "mean": 9.682042273616121, + "p50": 7.969379892329501, + "p90": 10.570340490525057, + "p95": 14.016753123090474, + "p99": 54.6038999076698 + }, + "itl_ms": { + "mean": 42.37590053159988, + "p50": 21.612419492157642, + "p90": 113.86398609756725, + "p95": 132.79422069972497, + "p99": 167.44969745966955 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_32_4000_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 7.7422533049975755, + "request_throughput": 16.532654636522523, + "input_token_throughput": 4552.67977053239, + "output_token_throughput": 2182.8270574783646, + "total_token_throughput": 6735.506828010754, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 1738.7919132729621, + "p50": 1538.9780115001486, + "p90": 2865.2426461980212, + "p95": 3830.7956773496635, + "p99": 5143.022389483815 + }, + "ttft_ms": { + "mean": 262.5563681242511, + "p50": 225.63749099936103, + "p90": 521.6669989036745, + "p95": 527.2058025482693, + "p99": 532.1059286133095 + }, + "tpot_ms": { + "mean": 12.059512611469799, + "p50": 10.357537249850187, + "p90": 20.459189923756977, + "p95": 22.732079964678167, + "p99": 28.16360305609708 + }, + "itl_ms": { + "mean": 48.537332998996746, + "p50": 27.78467499592807, + "p90": 105.31790559180081, + "p95": 108.22369960369542, + "p99": 175.24975506879855 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_32_512_256.jsonl" + } + ] +} \ No newline at end of file