results(max-context): complete H200 max context length exploration run 20260708-143534
This commit is contained in:
yy-fighting
parent
cd0b083a10
commit
a56c995091
@ -0,0 +1,36 @@
|
||||
# H200 Max Context Length Exploration Report
|
||||
|
||||
- Result root: `/data/user1/yy/experiments/dsv4_h200_max_context_length/results/20260708-143534`
|
||||
- Model: `/data/models/DeepSeek-V4-Flash`
|
||||
- Hardware: 8x NVIDIA H200 143GB
|
||||
- TP: 8
|
||||
- Output len: 1, num prompts: 1, concurrency: 1
|
||||
|
||||
## Per-attempt results
|
||||
|
||||
| Backend | Target len | Max model/context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| sglang | 65536 | 66560 | ✅ pass | 521.23 | 521.23 | 521.25 | 521.25 | |
|
||||
| sglang | 131072 | 132096 | ✅ pass | 668.67 | 668.67 | 668.69 | 668.69 | |
|
||||
| sglang | 262144 | 263168 | ✅ pass | 1232.63 | 1232.63 | 1232.65 | 1232.65 | |
|
||||
| sglang | 393216 | 394240 | ✅ pass | 1496.49 | 1496.49 | 1496.53 | 1496.53 | |
|
||||
| sglang | 524288 | 525312 | ✅ pass | 1764.18 | 1764.18 | 1764.23 | 1764.23 | |
|
||||
| sglang | 655360 | 656384 | ✅ pass | 2129.19 | 2129.19 | 2129.24 | 2129.24 | |
|
||||
| sglang | 786432 | 787456 | ✅ pass | 2440.34 | 2440.34 | 2440.35 | 2440.35 | |
|
||||
| sglang | 917504 | 918528 | ✅ pass | 3149.58 | 3149.58 | 3149.61 | 3149.61 | |
|
||||
| sglang | 1048576 | 1049600 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | server failed to start |
|
||||
| vllm | 65536 | 66560 | ✅ pass | 594.47 | 594.47 | 594.53 | 594.53 | |
|
||||
| vllm | 131072 | 132096 | ✅ pass | 889.46 | 889.46 | 889.51 | 889.51 | |
|
||||
| vllm | 262144 | 263168 | ✅ pass | 1178.53 | 1178.53 | 1178.56 | 1178.56 | |
|
||||
| vllm | 393216 | 394240 | ✅ pass | 1371.19 | 1371.19 | 1371.24 | 1371.24 | |
|
||||
| vllm | 524288 | 525312 | ✅ pass | 2582.85 | 2582.85 | 2582.90 | 2582.90 | |
|
||||
| vllm | 655360 | 656384 | ✅ pass | 2203.55 | 2203.55 | 2203.59 | 2203.59 | |
|
||||
| vllm | 786432 | 787456 | ✅ pass | 3886.52 | 3886.52 | 3886.57 | 3886.57 | |
|
||||
| vllm | 917504 | 918528 | ✅ pass | 3715.61 | 3715.61 | 3715.64 | 3715.64 | |
|
||||
| vllm | 1048576 | 1049600 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | server failed to start |
|
||||
|
||||
## Maximum supported input length
|
||||
|
||||
- **sglang**: 917504 tokens
|
||||
- **vllm**: 917504 tokens
|
||||
|
||||
@ -0,0 +1,783 @@
|
||||
{
|
||||
"metadata": {
|
||||
"experiment": "dsv4_h200_max_context_length",
|
||||
"run_id": "20260708-143534",
|
||||
"timestamp": "2026-07-08T14:35:34+00:00",
|
||||
"model": "/data/models/DeepSeek-V4-Flash",
|
||||
"backend": "sglang+vllm",
|
||||
"engine": "sglang+vllm",
|
||||
"hardware": "8x NVIDIA H200 143GB",
|
||||
"accelerator": "NVIDIA H200",
|
||||
"chip": "nvidia_h200",
|
||||
"script": "experiments/dsv4_h200_max_context_length/run_bench.sh",
|
||||
"env": "/data/user1/yy/envs/sglang",
|
||||
"git_commit": "ef65885",
|
||||
"git_dirty": "dirty",
|
||||
"description": "H200 max context length exploration for DeepSeek-V4-Flash"
|
||||
},
|
||||
"config": {
|
||||
"tp": 8,
|
||||
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"output_len": 1,
|
||||
"num_prompts": 1,
|
||||
"max_concurrency": 1,
|
||||
"request_rate": 10000,
|
||||
"context_pad": 1024
|
||||
},
|
||||
"scenarios": [
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 65536,
|
||||
"max_model_len": 66560,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 66560 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 0.5241995020041941,
|
||||
"request_throughput": 1.9076706409995772,
|
||||
"input_token_throughput": 125021.10312854829,
|
||||
"output_token_throughput": 1.9076706409995772,
|
||||
"total_input_tokens": 65536,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 521.2510440032929,
|
||||
"p50": 521.2510440032929,
|
||||
"p90": 521.2510440032929,
|
||||
"p95": 521.2510440032929,
|
||||
"p99": 521.2510440032929
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 521.2282620050246,
|
||||
"p50": 521.2282620050246,
|
||||
"p90": 521.2282620050246,
|
||||
"p95": 521.2282620050246,
|
||||
"p99": 521.2282620050246
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 131072,
|
||||
"max_model_len": 132096,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 132096 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 0.6736304189980729,
|
||||
"request_throughput": 1.484493532057763,
|
||||
"input_token_throughput": 194575.53623387511,
|
||||
"output_token_throughput": 1.484493532057763,
|
||||
"total_input_tokens": 131072,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 668.692392995581,
|
||||
"p50": 668.692392995581,
|
||||
"p90": 668.692392995581,
|
||||
"p95": 668.692392995581,
|
||||
"p99": 668.692392995581
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 668.666598998243,
|
||||
"p50": 668.666598998243,
|
||||
"p90": 668.666598998243,
|
||||
"p95": 668.666598998243,
|
||||
"p99": 668.666598998243
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 262144,
|
||||
"max_model_len": 263168,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 263168 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 1.235619597006007,
|
||||
"request_throughput": 0.8093105697118029,
|
||||
"input_token_throughput": 212155.90998653087,
|
||||
"output_token_throughput": 0.8093105697118029,
|
||||
"total_input_tokens": 262144,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 1232.6548010023544,
|
||||
"p50": 1232.6548010023544,
|
||||
"p90": 1232.6548010023544,
|
||||
"p95": 1232.6548010023544,
|
||||
"p99": 1232.6548010023544
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 1232.628379992093,
|
||||
"p50": 1232.628379992093,
|
||||
"p90": 1232.628379992093,
|
||||
"p95": 1232.628379992093,
|
||||
"p99": 1232.628379992093
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 393216,
|
||||
"max_model_len": 394240,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 394240 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 1.4997316709923325,
|
||||
"request_throughput": 0.6667859453406932,
|
||||
"input_token_throughput": 262190.90228308603,
|
||||
"output_token_throughput": 0.6667859453406932,
|
||||
"total_input_tokens": 393216,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 1496.5319500042824,
|
||||
"p50": 1496.5319500042824,
|
||||
"p90": 1496.5319500042824,
|
||||
"p95": 1496.5319500042824,
|
||||
"p99": 1496.5319500042824
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 1496.4891230047215,
|
||||
"p50": 1496.4891230047215,
|
||||
"p90": 1496.4891230047215,
|
||||
"p95": 1496.4891230047215,
|
||||
"p99": 1496.4891230047215
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 524288,
|
||||
"max_model_len": 525312,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 525312 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 1.7669223830016563,
|
||||
"request_throughput": 0.5659558165204717,
|
||||
"input_token_throughput": 296723.84313188505,
|
||||
"output_token_throughput": 0.5659558165204717,
|
||||
"total_input_tokens": 524288,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 1764.228992004064,
|
||||
"p50": 1764.228992004064,
|
||||
"p90": 1764.228992004064,
|
||||
"p95": 1764.228992004064,
|
||||
"p99": 1764.228992004064
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 1764.184692001436,
|
||||
"p50": 1764.184692001436,
|
||||
"p90": 1764.184692001436,
|
||||
"p95": 1764.184692001436,
|
||||
"p99": 1764.184692001436
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 655360,
|
||||
"max_model_len": 656384,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 656384 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 2.1324244450079277,
|
||||
"request_throughput": 0.4689497920271132,
|
||||
"input_token_throughput": 307330.93570288894,
|
||||
"output_token_throughput": 0.4689497920271132,
|
||||
"total_input_tokens": 655360,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 2129.2401870014146,
|
||||
"p50": 2129.2401870014146,
|
||||
"p90": 2129.2401870014146,
|
||||
"p95": 2129.2401870014146,
|
||||
"p99": 2129.2401870014146
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 2129.1923570097424,
|
||||
"p50": 2129.1923570097424,
|
||||
"p90": 2129.1923570097424,
|
||||
"p95": 2129.1923570097424,
|
||||
"p99": 2129.1923570097424
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 786432,
|
||||
"max_model_len": 787456,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 787456 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 2.4429005489946576,
|
||||
"request_throughput": 0.409349451581865,
|
||||
"input_token_throughput": 321925.50790642924,
|
||||
"output_token_throughput": 0.409349451581865,
|
||||
"total_input_tokens": 786432,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 2440.3524239896797,
|
||||
"p50": 2440.3524239896797,
|
||||
"p90": 2440.3524239896797,
|
||||
"p95": 2440.3524239896797,
|
||||
"p99": 2440.3524239896797
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 2440.341288995114,
|
||||
"p50": 2440.341288995114,
|
||||
"p90": 2440.341288995114,
|
||||
"p95": 2440.341288995114,
|
||||
"p99": 2440.341288995114
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 917504,
|
||||
"max_model_len": 918528,
|
||||
"success": true,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 918528 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 3.152248802012764,
|
||||
"request_throughput": 0.31723384250681075,
|
||||
"input_token_throughput": 291063.3194353689,
|
||||
"output_token_throughput": 0.31723384250681075,
|
||||
"total_input_tokens": 917504,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 3149.605149999843,
|
||||
"p50": 3149.605149999843,
|
||||
"p90": 3149.605149999843,
|
||||
"p95": 3149.605149999843,
|
||||
"p99": 3149.605149999843
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 3149.579901000834,
|
||||
"p50": 3149.579901000834,
|
||||
"p90": 3149.579901000834,
|
||||
"p95": 3149.579901000834,
|
||||
"p99": 3149.579901000834
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "sglang",
|
||||
"target_len": 1048576,
|
||||
"max_model_len": 1049600,
|
||||
"success": false,
|
||||
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 1049600 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
||||
"metrics": null,
|
||||
"error": "server failed to start"
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 65536,
|
||||
"max_model_len": 66560,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 66560 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 0.5968579519976629,
|
||||
"request_throughput": 1.675440524253777,
|
||||
"input_token_throughput": 109801.67019749554,
|
||||
"output_token_throughput": 1.675440524253777,
|
||||
"total_input_tokens": 65536,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 594.5277150021866,
|
||||
"p50": 594.5277150021866,
|
||||
"p90": 594.5277150021866,
|
||||
"p95": 594.5277150021866,
|
||||
"p99": 594.5277150021866
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 594.4739139958983,
|
||||
"p50": 594.4739139958983,
|
||||
"p90": 594.4739139958983,
|
||||
"p95": 594.4739139958983,
|
||||
"p99": 594.4739139958983
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 131072,
|
||||
"max_model_len": 132096,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 132096 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 0.8928074569994351,
|
||||
"request_throughput": 1.1200623294084255,
|
||||
"input_token_throughput": 146808.80964022115,
|
||||
"output_token_throughput": 1.1200623294084255,
|
||||
"total_input_tokens": 131072,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 889.5100749941776,
|
||||
"p50": 889.5100749941776,
|
||||
"p90": 889.5100749941776,
|
||||
"p95": 889.5100749941776,
|
||||
"p99": 889.5100749941776
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 889.4585100060795,
|
||||
"p50": 889.4585100060795,
|
||||
"p90": 889.4585100060795,
|
||||
"p95": 889.4585100060795,
|
||||
"p99": 889.4585100060795
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 262144,
|
||||
"max_model_len": 263168,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 263168 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 1.1815237270056969,
|
||||
"request_throughput": 0.8463647213706597,
|
||||
"input_token_throughput": 221869.4335189902,
|
||||
"output_token_throughput": 0.8463647213706597,
|
||||
"total_input_tokens": 262144,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 1178.564740010188,
|
||||
"p50": 1178.564740010188,
|
||||
"p90": 1178.564740010188,
|
||||
"p95": 1178.564740010188,
|
||||
"p99": 1178.564740010188
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 1178.5299420007505,
|
||||
"p50": 1178.5299420007505,
|
||||
"p90": 1178.5299420007505,
|
||||
"p95": 1178.5299420007505,
|
||||
"p99": 1178.5299420007505
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 393216,
|
||||
"max_model_len": 394240,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 394240 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 1.3780717330082553,
|
||||
"request_throughput": 0.725651630497532,
|
||||
"input_token_throughput": 285337.83153771755,
|
||||
"output_token_throughput": 0.725651630497532,
|
||||
"total_input_tokens": 393216,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 1371.242612003698,
|
||||
"p50": 1371.242612003698,
|
||||
"p90": 1371.242612003698,
|
||||
"p95": 1371.242612003698,
|
||||
"p99": 1371.242612003698
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 1371.1869800026761,
|
||||
"p50": 1371.1869800026761,
|
||||
"p90": 1371.1869800026761,
|
||||
"p95": 1371.1869800026761,
|
||||
"p99": 1371.1869800026761
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 524288,
|
||||
"max_model_len": 525312,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 525312 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 2.587135097011924,
|
||||
"request_throughput": 0.3865279401740462,
|
||||
"input_token_throughput": 202651.96069797032,
|
||||
"output_token_throughput": 0.3865279401740462,
|
||||
"total_input_tokens": 524288,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 2582.903844988323,
|
||||
"p50": 2582.903844988323,
|
||||
"p90": 2582.903844988323,
|
||||
"p95": 2582.903844988323,
|
||||
"p99": 2582.903844988323
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 2582.850620994577,
|
||||
"p50": 2582.850620994577,
|
||||
"p90": 2582.850620994577,
|
||||
"p95": 2582.850620994577,
|
||||
"p99": 2582.850620994577
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 655360,
|
||||
"max_model_len": 656384,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 656384 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 2.206327235006029,
|
||||
"request_throughput": 0.453241923561383,
|
||||
"input_token_throughput": 297036.62702518795,
|
||||
"output_token_throughput": 0.453241923561383,
|
||||
"total_input_tokens": 655360,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 2203.5920649941545,
|
||||
"p50": 2203.5920649941545,
|
||||
"p90": 2203.5920649941545,
|
||||
"p95": 2203.5920649941545,
|
||||
"p99": 2203.5920649941545
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 2203.5506129905116,
|
||||
"p50": 2203.5506129905116,
|
||||
"p90": 2203.5506129905116,
|
||||
"p95": 2203.5506129905116,
|
||||
"p99": 2203.5506129905116
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 786432,
|
||||
"max_model_len": 787456,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 787456 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 3.8894555929873604,
|
||||
"request_throughput": 0.25710539073977023,
|
||||
"input_token_throughput": 202195.90665025898,
|
||||
"output_token_throughput": 0.25710539073977023,
|
||||
"total_input_tokens": 786432,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 3886.5709000092465,
|
||||
"p50": 3886.5709000092465,
|
||||
"p90": 3886.5709000092465,
|
||||
"p95": 3886.5709000092465,
|
||||
"p99": 3886.5709000092465
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 3886.5172150108265,
|
||||
"p50": 3886.5172150108265,
|
||||
"p90": 3886.5172150108265,
|
||||
"p95": 3886.5172150108265,
|
||||
"p99": 3886.5172150108265
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 917504,
|
||||
"max_model_len": 918528,
|
||||
"success": true,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 918528 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": {
|
||||
"success_count": 1,
|
||||
"total_count": 1,
|
||||
"duration_s": 3.7180563760048244,
|
||||
"request_throughput": 0.26895772921940825,
|
||||
"input_token_throughput": 246769.79238972397,
|
||||
"output_token_throughput": 0.26895772921940825,
|
||||
"total_input_tokens": 917504,
|
||||
"total_output_tokens": 1,
|
||||
"e2e_ms": {
|
||||
"mean": 3715.635304004536,
|
||||
"p50": 3715.635304004536,
|
||||
"p90": 3715.635304004536,
|
||||
"p95": 3715.635304004536,
|
||||
"p99": 3715.635304004536
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 3715.6081900029676,
|
||||
"p50": 3715.6081900029676,
|
||||
"p90": 3715.6081900029676,
|
||||
"p95": 3715.6081900029676,
|
||||
"p99": 3715.6081900029676
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 0.0,
|
||||
"p50": 0.0,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 0.0
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"backend": "vllm",
|
||||
"target_len": 1048576,
|
||||
"max_model_len": 1049600,
|
||||
"success": false,
|
||||
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 1049600 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
||||
"metrics": null,
|
||||
"error": "server failed to start"
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user