results(vllm-dspark-vs-default): complete TTFT comparison run 20260708-142121
This commit is contained in:
yy-fighting
parent
27ceaac47f
commit
cd0b083a10
@ -0,0 +1,30 @@
|
|||||||
|
# vLLM+DSpark vs vLLM default on DeepSeek-V4-Flash (H200, TP=8)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark`
|
||||||
|
- Default model: `/data/models/DeepSeek-V4-Flash`
|
||||||
|
- Hardware: 8x NVIDIA H200 143GB
|
||||||
|
- Tensor Parallelism: 8
|
||||||
|
- Benchmark client: `sglang.bench_serving --backend vllm`
|
||||||
|
- DSpark flags: `--spec-method dspark --spec-model <model> --spec-tokens 5`
|
||||||
|
|
||||||
|
## Side-by-side results (TTFT focused)
|
||||||
|
|
||||||
|
| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) |
|
||||||
|
|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||||
|
| c128_i512_o256 | vllm-dspark | 128 | 512 | 256 | 23.94 | 3160.81 | 778.36 | 1071.85 | 1075.35 | 28.95 | 78.75 | 3632.34 |
|
||||||
|
| c128_i512_o256 | vllm-default | 128 | 512 | 256 | 29.99 | 3959.89 | 617.72 | 767.18 | 770.89 | 18.31 | 65.44 | 2695.91 |
|
||||||
|
| c1_i4000_o512 | vllm-dspark | 1 | 4000 | 512 | 1.73 | 486.20 | 123.58 | 176.29 | 190.52 | 1.66 | 3.56 | 578.26 |
|
||||||
|
| c1_i4000_o512 | vllm-default | 1 | 4000 | 512 | 0.50 | 140.13 | 131.50 | 191.53 | 268.69 | 6.68 | 6.73 | 2008.09 |
|
||||||
|
| c1_i512_o256 | vllm-dspark | 1 | 512 | 256 | 2.56 | 372.62 | 86.51 | 185.00 | 188.81 | 2.09 | 3.83 | 389.62 |
|
||||||
|
| c1_i512_o256 | vllm-default | 1 | 512 | 256 | 0.99 | 143.82 | 49.50 | 68.50 | 71.09 | 6.66 | 6.71 | 1010.98 |
|
||||||
|
| c32_i4000_o512 | vllm-dspark | 32 | 4000 | 512 | 11.49 | 3242.88 | 298.42 | 434.24 | 522.34 | 9.68 | 54.60 | 2476.82 |
|
||||||
|
| c32_i4000_o512 | vllm-default | 32 | 4000 | 512 | 5.38 | 1518.49 | 382.74 | 681.98 | 684.75 | 16.43 | 25.49 | 4927.71 |
|
||||||
|
| c32_i512_o256 | vllm-dspark | 32 | 512 | 256 | 16.53 | 2182.83 | 262.56 | 527.21 | 532.11 | 12.06 | 28.16 | 1738.79 |
|
||||||
|
| c32_i512_o256 | vllm-default | 32 | 512 | 256 | 12.98 | 1713.20 | 122.62 | 272.00 | 304.80 | 15.77 | 23.26 | 2154.65 |
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- TTFT mean/P95/P99 are the main focus for verifying whether DSpark increases time-to-first-token.
|
||||||
|
- Mean TPOT and E2E are included to check whether speculative decoding pays back after first token.
|
||||||
@ -0,0 +1,18 @@
|
|||||||
|
# H200 vLLM default Benchmark Report
|
||||||
|
|
||||||
|
- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default`
|
||||||
|
- Backend: vLLM default (TP=8)
|
||||||
|
- Benchmark client: `sglang.bench_serving --backend vllm`
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|
||||||
|
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||||
|
| c128_i512_o256 | 128 | 512 | 256 | 4.27 | 128 | 29.99 | 8259.06 | 3959.89 | 12218.95 | 617.72 | 767.18 | 770.89 | 18.31 | 28.68 | 65.44 | 2695.91 | 4153.36 | 4205.54 | ✅ |
|
||||||
|
| c1_i4000_o512 | 1 | 4000 | 512 | 64.29 | 32 | 0.50 | 1011.09 | 140.13 | 1151.23 | 131.50 | 191.53 | 268.69 | 6.68 | 6.73 | 6.73 | 2008.09 | 3523.51 | 3599.32 | ✅ |
|
||||||
|
| c1_i512_o256 | 1 | 512 | 256 | 32.38 | 32 | 0.99 | 252.30 | 143.82 | 396.12 | 49.50 | 68.50 | 71.09 | 6.66 | 6.69 | 6.71 | 1010.98 | 1735.31 | 1759.78 | ✅ |
|
||||||
|
| c32_i4000_o512 | 32 | 4000 | 512 | 11.90 | 64 | 5.38 | 11297.00 | 1518.49 | 12815.49 | 382.74 | 681.98 | 684.75 | 16.43 | 20.90 | 25.49 | 4927.71 | 8743.37 | 9349.37 | ✅ |
|
||||||
|
| c32_i512_o256 | 32 | 512 | 256 | 9.86 | 128 | 12.98 | 3573.19 | 1713.20 | 5286.40 | 122.62 | 272.00 | 304.80 | 15.77 | 19.50 | 23.26 | 2154.65 | 4102.71 | 4275.05 | ✅ |
|
||||||
|
|
||||||
|
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||||
|
|
||||||
@ -0,0 +1,304 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "dsv4_h200_vllm_dspark_vs_default_default",
|
||||||
|
"run_id": "20260708-142121",
|
||||||
|
"timestamp": "2026-07-08T14:21:25+00:00",
|
||||||
|
"model": "/data/models/DeepSeek-V4-Flash",
|
||||||
|
"backend": "vllm",
|
||||||
|
"engine": "vllm-default",
|
||||||
|
"hardware": "8x NVIDIA H200 143GB",
|
||||||
|
"accelerator": "NVIDIA H200",
|
||||||
|
"chip": "nvidia_h200",
|
||||||
|
"script": "experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh",
|
||||||
|
"env": "/data/user1/yy/envs/vllm",
|
||||||
|
"git_commit": "1a6b173",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 vllm-default TP=8 benchmark for TTFT comparison"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"tp": 8,
|
||||||
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
||||||
|
"max_model_len": 32768,
|
||||||
|
"max_num_seqs": 256,
|
||||||
|
"backend": "default",
|
||||||
|
"server_start_script": "experiments/dsv4_h200_vllm_dspark_vs_default/start_default.sh",
|
||||||
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30004"
|
||||||
|
},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c128_i512_o256",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 128,
|
||||||
|
"input_len": 512,
|
||||||
|
"output_len": 256,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 128
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 128,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 4.267798511005822,
|
||||||
|
"request_throughput": 29.992043830071385,
|
||||||
|
"input_token_throughput": 8259.059069705907,
|
||||||
|
"output_token_throughput": 3959.8870369391125,
|
||||||
|
"total_token_throughput": 12218.94610664502,
|
||||||
|
"total_input_tokens": 35248,
|
||||||
|
"total_output_tokens": 16900,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 2695.9114590795252,
|
||||||
|
"p50": 2786.5102920040954,
|
||||||
|
"p90": 4094.3853520046105,
|
||||||
|
"p95": 4153.361671900348,
|
||||||
|
"p99": 4205.538639774604
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 617.7214959924413,
|
||||||
|
"p50": 563.3532095089322,
|
||||||
|
"p90": 764.0312482006266,
|
||||||
|
"p95": 767.1767810017627,
|
||||||
|
"p99": 770.8853784023086
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 18.314879062516017,
|
||||||
|
"p50": 15.941864871586276,
|
||||||
|
"p90": 23.19956007269117,
|
||||||
|
"p95": 28.68311282203317,
|
||||||
|
"p99": 65.43692430894995
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 15.86385086378528,
|
||||||
|
"p50": 14.933839993318543,
|
||||||
|
"p90": 17.340479693666566,
|
||||||
|
"p95": 18.231574493984226,
|
||||||
|
"p99": 21.184932928154012
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_128_512_256.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4000_o512",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 1,
|
||||||
|
"input_len": 4000,
|
||||||
|
"output_len": 512,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 32
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 32,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 64.28883961599786,
|
||||||
|
"request_throughput": 0.49775357886591887,
|
||||||
|
"input_token_throughput": 1011.0930666700768,
|
||||||
|
"output_token_throughput": 140.13318725009572,
|
||||||
|
"total_token_throughput": 1151.2262539201727,
|
||||||
|
"total_input_tokens": 65002,
|
||||||
|
"total_output_tokens": 9009,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 2008.0858264368544,
|
||||||
|
"p50": 2064.2024725020747,
|
||||||
|
"p90": 3374.621986798593,
|
||||||
|
"p95": 3523.513536003884,
|
||||||
|
"p99": 3599.319600486051
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 131.49705237538,
|
||||||
|
"p50": 128.75337050354574,
|
||||||
|
"p90": 188.4824504086282,
|
||||||
|
"p95": 191.5273676990182,
|
||||||
|
"p99": 268.6876055170434
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 6.678438385450706,
|
||||||
|
"p50": 6.647801321133651,
|
||||||
|
"p90": 6.729117710662969,
|
||||||
|
"p95": 6.730015678053446,
|
||||||
|
"p99": 6.730549147956696
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 6.689239366715674,
|
||||||
|
"p50": 6.692727998597547,
|
||||||
|
"p90": 6.8432597909122705,
|
||||||
|
"p95": 6.884838393307291,
|
||||||
|
"p99": 6.988287522690371
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_1_4000_512.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i512_o256",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 1,
|
||||||
|
"input_len": 512,
|
||||||
|
"output_len": 256,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 32
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 32,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 32.381663249005214,
|
||||||
|
"request_throughput": 0.9882135995896709,
|
||||||
|
"input_token_throughput": 252.30328464523785,
|
||||||
|
"output_token_throughput": 143.81596041528428,
|
||||||
|
"total_token_throughput": 396.11924506052213,
|
||||||
|
"total_input_tokens": 8170,
|
||||||
|
"total_output_tokens": 4657,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 1010.982028125909,
|
||||||
|
"p50": 1195.345941996493,
|
||||||
|
"p90": 1688.3191696979338,
|
||||||
|
"p95": 1735.3133527984028,
|
||||||
|
"p99": 1759.781412960583
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 49.50495168895941,
|
||||||
|
"p50": 44.729964502039365,
|
||||||
|
"p90": 65.38926539797103,
|
||||||
|
"p95": 68.496801002766,
|
||||||
|
"p99": 71.08588116068859
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 6.659676006644546,
|
||||||
|
"p50": 6.654429546349644,
|
||||||
|
"p90": 6.684645557662856,
|
||||||
|
"p95": 6.69398312746385,
|
||||||
|
"p99": 6.711085371917718
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 6.652042307463047,
|
||||||
|
"p50": 6.637780999881215,
|
||||||
|
"p90": 6.787886397796684,
|
||||||
|
"p95": 6.841546000214294,
|
||||||
|
"p99": 7.143319878960029
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_1_512_256.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i4000_o512",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 32,
|
||||||
|
"input_len": 4000,
|
||||||
|
"output_len": 512,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 64
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 64,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 11.899348942009965,
|
||||||
|
"request_throughput": 5.378445519321792,
|
||||||
|
"input_token_throughput": 11297.004622279226,
|
||||||
|
"output_token_throughput": 1518.4864388847727,
|
||||||
|
"total_token_throughput": 12815.491061163999,
|
||||||
|
"total_input_tokens": 134427,
|
||||||
|
"total_output_tokens": 18069,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 4927.714462639642,
|
||||||
|
"p50": 4734.132497498649,
|
||||||
|
"p90": 8425.151473002916,
|
||||||
|
"p95": 8743.374133249745,
|
||||||
|
"p99": 9349.369201347727
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 382.74424496808024,
|
||||||
|
"p50": 390.3064074984286,
|
||||||
|
"p90": 678.5326104989508,
|
||||||
|
"p95": 681.9790261564776,
|
||||||
|
"p99": 684.7464643983403
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 16.425838503257715,
|
||||||
|
"p50": 17.044603306082735,
|
||||||
|
"p90": 18.812656380989317,
|
||||||
|
"p95": 20.897049261926014,
|
||||||
|
"p99": 25.487691384272917
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 16.239184021720266,
|
||||||
|
"p50": 11.25292000506306,
|
||||||
|
"p90": 12.360760505544022,
|
||||||
|
"p95": 48.09924705259619,
|
||||||
|
"p99": 161.26204263797263
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_32_4000_512.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i512_o256",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 32,
|
||||||
|
"input_len": 512,
|
||||||
|
"output_len": 256,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 128
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 128,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 9.864565095995204,
|
||||||
|
"request_throughput": 12.975736766333995,
|
||||||
|
"input_token_throughput": 3573.1935120292237,
|
||||||
|
"output_token_throughput": 1713.2027449300354,
|
||||||
|
"total_token_throughput": 5286.3962569592595,
|
||||||
|
"total_input_tokens": 35248,
|
||||||
|
"total_output_tokens": 16900,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 2154.6506077650065,
|
||||||
|
"p50": 2170.227252499899,
|
||||||
|
"p90": 3869.9233716019075,
|
||||||
|
"p95": 4102.707951706542,
|
||||||
|
"p99": 4275.054011319589
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 122.61577058620787,
|
||||||
|
"p50": 76.25793749321019,
|
||||||
|
"p90": 266.052756505087,
|
||||||
|
"p95": 272.00344715092797,
|
||||||
|
"p99": 304.79746079756296
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 15.771494928385172,
|
||||||
|
"p50": 16.114622466954064,
|
||||||
|
"p90": 17.704943566836604,
|
||||||
|
"p95": 19.49543390091434,
|
||||||
|
"p99": 23.258708736311746
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 15.513364437561801,
|
||||||
|
"p50": 11.447760996816214,
|
||||||
|
"p90": 26.607878498907667,
|
||||||
|
"p95": 30.321172751428094,
|
||||||
|
"p99": 100.95071544419625
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default/raw_outputs/default_0708_32_512_256.jsonl"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,18 @@
|
|||||||
|
# H200 vLLM+DSpark Benchmark Report
|
||||||
|
|
||||||
|
- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark`
|
||||||
|
- Backend: vLLM+DSpark (TP=8)
|
||||||
|
- Benchmark client: `sglang.bench_serving --backend vllm`
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|
||||||
|
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||||
|
| c128_i512_o256 | 128 | 512 | 256 | 5.35 | 128 | 23.94 | 6592.44 | 3160.81 | 9753.24 | 778.36 | 1071.85 | 1075.35 | 28.95 | 69.00 | 78.75 | 3632.34 | 5022.66 | 5229.14 | ✅ |
|
||||||
|
| c1_i4000_o512 | 1 | 4000 | 512 | 18.53 | 32 | 1.73 | 3508.05 | 486.20 | 3994.25 | 123.58 | 176.29 | 190.52 | 1.66 | 2.30 | 3.56 | 578.26 | 964.23 | 1147.23 | ✅ |
|
||||||
|
| c1_i512_o256 | 1 | 512 | 256 | 12.50 | 32 | 2.56 | 653.70 | 372.62 | 1026.32 | 86.51 | 185.00 | 188.81 | 2.09 | 3.41 | 3.83 | 389.62 | 777.58 | 915.24 | ✅ |
|
||||||
|
| c32_i4000_o512 | 32 | 4000 | 512 | 5.57 | 64 | 11.49 | 24125.91 | 3242.88 | 27368.79 | 298.42 | 434.24 | 522.34 | 9.68 | 14.02 | 54.60 | 2476.82 | 4615.41 | 4710.26 | ✅ |
|
||||||
|
| c32_i512_o256 | 32 | 512 | 256 | 7.74 | 128 | 16.53 | 4552.68 | 2182.83 | 6735.51 | 262.56 | 527.21 | 532.11 | 12.06 | 22.73 | 28.16 | 1738.79 | 3830.80 | 5143.02 | ✅ |
|
||||||
|
|
||||||
|
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||||
|
|
||||||
@ -0,0 +1,304 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "dsv4_h200_vllm_dspark_vs_default_dspark",
|
||||||
|
"run_id": "20260708-142121",
|
||||||
|
"timestamp": "2026-07-08T14:21:25+00:00",
|
||||||
|
"model": "/data/models/DeepSeek-V4-Flash-DSpark",
|
||||||
|
"backend": "vllm",
|
||||||
|
"engine": "vllm-dspark",
|
||||||
|
"hardware": "8x NVIDIA H200 143GB",
|
||||||
|
"accelerator": "NVIDIA H200",
|
||||||
|
"chip": "nvidia_h200",
|
||||||
|
"script": "experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh",
|
||||||
|
"env": "/data/user1/yy/envs/vllm-dspark",
|
||||||
|
"git_commit": "1a6b173",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 vllm-dspark TP=8 benchmark for TTFT comparison"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"tp": 8,
|
||||||
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
||||||
|
"max_model_len": 32768,
|
||||||
|
"max_num_seqs": 256,
|
||||||
|
"backend": "dspark",
|
||||||
|
"server_start_script": "experiments/dsv4_h200_vllm_dspark_vs_default/start_dspark.sh",
|
||||||
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash-DSpark --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --spec-method dspark --spec-model /data/models/DeepSeek-V4-Flash-DSpark --spec-tokens 5 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30007"
|
||||||
|
},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c128_i512_o256",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 128,
|
||||||
|
"input_len": 512,
|
||||||
|
"output_len": 256,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 128
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 128,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 5.346734229999129,
|
||||||
|
"request_throughput": 23.939847109255112,
|
||||||
|
"input_token_throughput": 6592.435397711127,
|
||||||
|
"output_token_throughput": 3160.807938643839,
|
||||||
|
"total_token_throughput": 9753.243336354966,
|
||||||
|
"total_input_tokens": 35248,
|
||||||
|
"total_output_tokens": 16900,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 3632.342373304141,
|
||||||
|
"p50": 4026.272206501744,
|
||||||
|
"p90": 4568.2280586028355,
|
||||||
|
"p95": 5022.660237102536,
|
||||||
|
"p99": 5229.135414937482
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 778.3609630697583,
|
||||||
|
"p50": 860.7529874934698,
|
||||||
|
"p90": 1069.5359163975809,
|
||||||
|
"p95": 1071.847062408051,
|
||||||
|
"p99": 1075.3475757954584
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 28.950386059739248,
|
||||||
|
"p50": 25.412556098403837,
|
||||||
|
"p90": 47.2479180524213,
|
||||||
|
"p95": 69.00331620594505,
|
||||||
|
"p99": 78.74700601221775
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 93.81264048253927,
|
||||||
|
"p50": 40.38110149849672,
|
||||||
|
"p90": 167.10851929528872,
|
||||||
|
"p95": 178.33137240304495,
|
||||||
|
"p99": 226.998560061038
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_128_512_256.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4000_o512",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 1,
|
||||||
|
"input_len": 4000,
|
||||||
|
"output_len": 512,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 32
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 32,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 18.529377975006355,
|
||||||
|
"request_throughput": 1.7269872762681893,
|
||||||
|
"input_token_throughput": 3508.050841624526,
|
||||||
|
"output_token_throughput": 486.2008866218786,
|
||||||
|
"total_token_throughput": 3994.2517282464046,
|
||||||
|
"total_input_tokens": 65002,
|
||||||
|
"total_output_tokens": 9009,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 578.2551125621467,
|
||||||
|
"p50": 591.3451755041024,
|
||||||
|
"p90": 892.7950516954298,
|
||||||
|
"p95": 964.232088495919,
|
||||||
|
"p99": 1147.2280103387313
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 123.576120467078,
|
||||||
|
"p50": 116.67309649055824,
|
||||||
|
"p90": 165.92405038827567,
|
||||||
|
"p95": 176.29006390707215,
|
||||||
|
"p99": 190.51729560960666
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 1.663761482756609,
|
||||||
|
"p50": 1.5393092801409114,
|
||||||
|
"p90": 1.721910965739694,
|
||||||
|
"p95": 2.2984976072946695,
|
||||||
|
"p99": 3.557179712464779
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 8.58824287840908,
|
||||||
|
"p50": 8.61248900037026,
|
||||||
|
"p90": 8.832874392101076,
|
||||||
|
"p95": 8.92088470282033,
|
||||||
|
"p99": 9.17154152484727
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_1_4000_512.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i512_o256",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 1,
|
||||||
|
"input_len": 512,
|
||||||
|
"output_len": 256,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 32
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 32,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 12.498076534990105,
|
||||||
|
"request_throughput": 2.560393986259529,
|
||||||
|
"input_token_throughput": 653.7005896168861,
|
||||||
|
"output_token_throughput": 372.61733731283215,
|
||||||
|
"total_token_throughput": 1026.3179269297182,
|
||||||
|
"total_input_tokens": 8170,
|
||||||
|
"total_output_tokens": 4657,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 389.62139918930916,
|
||||||
|
"p50": 377.8842995016021,
|
||||||
|
"p90": 646.4735688015936,
|
||||||
|
"p95": 777.5833995023276,
|
||||||
|
"p99": 915.2371472108645
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 86.50895553273585,
|
||||||
|
"p50": 65.89411200548057,
|
||||||
|
"p90": 180.50938470114488,
|
||||||
|
"p95": 184.9997164463275,
|
||||||
|
"p99": 188.8067441045132
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 2.0946395465617687,
|
||||||
|
"p50": 1.810156284521167,
|
||||||
|
"p90": 3.2686956743572244,
|
||||||
|
"p95": 3.410268214368921,
|
||||||
|
"p99": 3.8341900713573533
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 8.582383006246063,
|
||||||
|
"p50": 8.498353497998323,
|
||||||
|
"p90": 8.866577497974504,
|
||||||
|
"p95": 9.046812405722449,
|
||||||
|
"p99": 11.975977418769617
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_1_512_256.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i4000_o512",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 32,
|
||||||
|
"input_len": 4000,
|
||||||
|
"output_len": 512,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 64
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 64,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 5.57189327001106,
|
||||||
|
"request_throughput": 11.486221450877318,
|
||||||
|
"input_token_throughput": 24125.910796516957,
|
||||||
|
"output_token_throughput": 3242.883365560973,
|
||||||
|
"total_token_throughput": 27368.794162077927,
|
||||||
|
"total_input_tokens": 134427,
|
||||||
|
"total_output_tokens": 18069,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 2476.816028921121,
|
||||||
|
"p50": 2355.44883399416,
|
||||||
|
"p90": 4230.580038897461,
|
||||||
|
"p95": 4615.411171296,
|
||||||
|
"p99": 4710.255208882008
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 298.4201381239018,
|
||||||
|
"p50": 329.9452685023425,
|
||||||
|
"p90": 424.3153512070421,
|
||||||
|
"p95": 434.23709184717154,
|
||||||
|
"p99": 522.339181041898
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 9.682042273616121,
|
||||||
|
"p50": 7.969379892329501,
|
||||||
|
"p90": 10.570340490525057,
|
||||||
|
"p95": 14.016753123090474,
|
||||||
|
"p99": 54.6038999076698
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 42.37590053159988,
|
||||||
|
"p50": 21.612419492157642,
|
||||||
|
"p90": 113.86398609756725,
|
||||||
|
"p95": 132.79422069972497,
|
||||||
|
"p99": 167.44969745966955
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_32_4000_512.jsonl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i512_o256",
|
||||||
|
"config": {
|
||||||
|
"concurrency": 32,
|
||||||
|
"input_len": 512,
|
||||||
|
"output_len": 256,
|
||||||
|
"dataset": "random",
|
||||||
|
"num_prompts": 128
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"success": 128,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 7.7422533049975755,
|
||||||
|
"request_throughput": 16.532654636522523,
|
||||||
|
"input_token_throughput": 4552.67977053239,
|
||||||
|
"output_token_throughput": 2182.8270574783646,
|
||||||
|
"total_token_throughput": 6735.506828010754,
|
||||||
|
"total_input_tokens": 35248,
|
||||||
|
"total_output_tokens": 16900,
|
||||||
|
"e2e_ms": {
|
||||||
|
"mean": 1738.7919132729621,
|
||||||
|
"p50": 1538.9780115001486,
|
||||||
|
"p90": 2865.2426461980212,
|
||||||
|
"p95": 3830.7956773496635,
|
||||||
|
"p99": 5143.022389483815
|
||||||
|
},
|
||||||
|
"ttft_ms": {
|
||||||
|
"mean": 262.5563681242511,
|
||||||
|
"p50": 225.63749099936103,
|
||||||
|
"p90": 521.6669989036745,
|
||||||
|
"p95": 527.2058025482693,
|
||||||
|
"p99": 532.1059286133095
|
||||||
|
},
|
||||||
|
"tpot_ms": {
|
||||||
|
"mean": 12.059512611469799,
|
||||||
|
"p50": 10.357537249850187,
|
||||||
|
"p90": 20.459189923756977,
|
||||||
|
"p95": 22.732079964678167,
|
||||||
|
"p99": 28.16360305609708
|
||||||
|
},
|
||||||
|
"itl_ms": {
|
||||||
|
"mean": 48.537332998996746,
|
||||||
|
"p50": 27.78467499592807,
|
||||||
|
"p90": 105.31790559180081,
|
||||||
|
"p95": 108.22369960369542,
|
||||||
|
"p99": 175.24975506879855
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"slo_status": {
|
||||||
|
"ttft_p95_ok": true,
|
||||||
|
"tpot_mean_ok": true,
|
||||||
|
"overall": "✅"
|
||||||
|
},
|
||||||
|
"raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/dspark/raw_outputs/dspark_0708_32_512_256.jsonl"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user