783 lines
25 KiB
JSON
783 lines
25 KiB
JSON
{
|
|
"metadata": {
|
|
"experiment": "dsv4_h200_max_context_length",
|
|
"run_id": "20260708-143534",
|
|
"timestamp": "2026-07-08T14:35:34+00:00",
|
|
"model": "/data/models/DeepSeek-V4-Flash",
|
|
"backend": "sglang+vllm",
|
|
"engine": "sglang+vllm",
|
|
"hardware": "8x NVIDIA H200 143GB",
|
|
"accelerator": "NVIDIA H200",
|
|
"chip": "nvidia_h200",
|
|
"script": "experiments/dsv4_h200_max_context_length/run_bench.sh",
|
|
"env": "/data/user1/yy/envs/sglang",
|
|
"git_commit": "ef65885",
|
|
"git_dirty": "dirty",
|
|
"description": "H200 max context length exploration for DeepSeek-V4-Flash"
|
|
},
|
|
"config": {
|
|
"tp": 8,
|
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
|
"output_len": 1,
|
|
"num_prompts": 1,
|
|
"max_concurrency": 1,
|
|
"request_rate": 10000,
|
|
"context_pad": 1024
|
|
},
|
|
"scenarios": [
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 65536,
|
|
"max_model_len": 66560,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 66560 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 0.5241995020041941,
|
|
"request_throughput": 1.9076706409995772,
|
|
"input_token_throughput": 125021.10312854829,
|
|
"output_token_throughput": 1.9076706409995772,
|
|
"total_input_tokens": 65536,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 521.2510440032929,
|
|
"p50": 521.2510440032929,
|
|
"p90": 521.2510440032929,
|
|
"p95": 521.2510440032929,
|
|
"p99": 521.2510440032929
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 521.2282620050246,
|
|
"p50": 521.2282620050246,
|
|
"p90": 521.2282620050246,
|
|
"p95": 521.2282620050246,
|
|
"p99": 521.2282620050246
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 131072,
|
|
"max_model_len": 132096,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 132096 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 0.6736304189980729,
|
|
"request_throughput": 1.484493532057763,
|
|
"input_token_throughput": 194575.53623387511,
|
|
"output_token_throughput": 1.484493532057763,
|
|
"total_input_tokens": 131072,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 668.692392995581,
|
|
"p50": 668.692392995581,
|
|
"p90": 668.692392995581,
|
|
"p95": 668.692392995581,
|
|
"p99": 668.692392995581
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 668.666598998243,
|
|
"p50": 668.666598998243,
|
|
"p90": 668.666598998243,
|
|
"p95": 668.666598998243,
|
|
"p99": 668.666598998243
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 262144,
|
|
"max_model_len": 263168,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 263168 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 1.235619597006007,
|
|
"request_throughput": 0.8093105697118029,
|
|
"input_token_throughput": 212155.90998653087,
|
|
"output_token_throughput": 0.8093105697118029,
|
|
"total_input_tokens": 262144,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 1232.6548010023544,
|
|
"p50": 1232.6548010023544,
|
|
"p90": 1232.6548010023544,
|
|
"p95": 1232.6548010023544,
|
|
"p99": 1232.6548010023544
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 1232.628379992093,
|
|
"p50": 1232.628379992093,
|
|
"p90": 1232.628379992093,
|
|
"p95": 1232.628379992093,
|
|
"p99": 1232.628379992093
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 393216,
|
|
"max_model_len": 394240,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 394240 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 1.4997316709923325,
|
|
"request_throughput": 0.6667859453406932,
|
|
"input_token_throughput": 262190.90228308603,
|
|
"output_token_throughput": 0.6667859453406932,
|
|
"total_input_tokens": 393216,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 1496.5319500042824,
|
|
"p50": 1496.5319500042824,
|
|
"p90": 1496.5319500042824,
|
|
"p95": 1496.5319500042824,
|
|
"p99": 1496.5319500042824
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 1496.4891230047215,
|
|
"p50": 1496.4891230047215,
|
|
"p90": 1496.4891230047215,
|
|
"p95": 1496.4891230047215,
|
|
"p99": 1496.4891230047215
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 524288,
|
|
"max_model_len": 525312,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 525312 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 1.7669223830016563,
|
|
"request_throughput": 0.5659558165204717,
|
|
"input_token_throughput": 296723.84313188505,
|
|
"output_token_throughput": 0.5659558165204717,
|
|
"total_input_tokens": 524288,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 1764.228992004064,
|
|
"p50": 1764.228992004064,
|
|
"p90": 1764.228992004064,
|
|
"p95": 1764.228992004064,
|
|
"p99": 1764.228992004064
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 1764.184692001436,
|
|
"p50": 1764.184692001436,
|
|
"p90": 1764.184692001436,
|
|
"p95": 1764.184692001436,
|
|
"p99": 1764.184692001436
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 655360,
|
|
"max_model_len": 656384,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 656384 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 2.1324244450079277,
|
|
"request_throughput": 0.4689497920271132,
|
|
"input_token_throughput": 307330.93570288894,
|
|
"output_token_throughput": 0.4689497920271132,
|
|
"total_input_tokens": 655360,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 2129.2401870014146,
|
|
"p50": 2129.2401870014146,
|
|
"p90": 2129.2401870014146,
|
|
"p95": 2129.2401870014146,
|
|
"p99": 2129.2401870014146
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 2129.1923570097424,
|
|
"p50": 2129.1923570097424,
|
|
"p90": 2129.1923570097424,
|
|
"p95": 2129.1923570097424,
|
|
"p99": 2129.1923570097424
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 786432,
|
|
"max_model_len": 787456,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 787456 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 2.4429005489946576,
|
|
"request_throughput": 0.409349451581865,
|
|
"input_token_throughput": 321925.50790642924,
|
|
"output_token_throughput": 0.409349451581865,
|
|
"total_input_tokens": 786432,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 2440.3524239896797,
|
|
"p50": 2440.3524239896797,
|
|
"p90": 2440.3524239896797,
|
|
"p95": 2440.3524239896797,
|
|
"p99": 2440.3524239896797
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 2440.341288995114,
|
|
"p50": 2440.341288995114,
|
|
"p90": 2440.341288995114,
|
|
"p95": 2440.341288995114,
|
|
"p99": 2440.341288995114
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 917504,
|
|
"max_model_len": 918528,
|
|
"success": true,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 918528 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 3.152248802012764,
|
|
"request_throughput": 0.31723384250681075,
|
|
"input_token_throughput": 291063.3194353689,
|
|
"output_token_throughput": 0.31723384250681075,
|
|
"total_input_tokens": 917504,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 3149.605149999843,
|
|
"p50": 3149.605149999843,
|
|
"p90": 3149.605149999843,
|
|
"p95": 3149.605149999843,
|
|
"p99": 3149.605149999843
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 3149.579901000834,
|
|
"p50": 3149.579901000834,
|
|
"p90": 3149.579901000834,
|
|
"p95": 3149.579901000834,
|
|
"p99": 3149.579901000834
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 1048576,
|
|
"max_model_len": 1049600,
|
|
"success": false,
|
|
"server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 1049600 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
|
|
"metrics": null,
|
|
"error": "server failed to start"
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 65536,
|
|
"max_model_len": 66560,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 66560 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 0.5968579519976629,
|
|
"request_throughput": 1.675440524253777,
|
|
"input_token_throughput": 109801.67019749554,
|
|
"output_token_throughput": 1.675440524253777,
|
|
"total_input_tokens": 65536,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 594.5277150021866,
|
|
"p50": 594.5277150021866,
|
|
"p90": 594.5277150021866,
|
|
"p95": 594.5277150021866,
|
|
"p99": 594.5277150021866
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 594.4739139958983,
|
|
"p50": 594.4739139958983,
|
|
"p90": 594.4739139958983,
|
|
"p95": 594.4739139958983,
|
|
"p99": 594.4739139958983
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 131072,
|
|
"max_model_len": 132096,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 132096 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 0.8928074569994351,
|
|
"request_throughput": 1.1200623294084255,
|
|
"input_token_throughput": 146808.80964022115,
|
|
"output_token_throughput": 1.1200623294084255,
|
|
"total_input_tokens": 131072,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 889.5100749941776,
|
|
"p50": 889.5100749941776,
|
|
"p90": 889.5100749941776,
|
|
"p95": 889.5100749941776,
|
|
"p99": 889.5100749941776
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 889.4585100060795,
|
|
"p50": 889.4585100060795,
|
|
"p90": 889.4585100060795,
|
|
"p95": 889.4585100060795,
|
|
"p99": 889.4585100060795
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 262144,
|
|
"max_model_len": 263168,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 263168 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 1.1815237270056969,
|
|
"request_throughput": 0.8463647213706597,
|
|
"input_token_throughput": 221869.4335189902,
|
|
"output_token_throughput": 0.8463647213706597,
|
|
"total_input_tokens": 262144,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 1178.564740010188,
|
|
"p50": 1178.564740010188,
|
|
"p90": 1178.564740010188,
|
|
"p95": 1178.564740010188,
|
|
"p99": 1178.564740010188
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 1178.5299420007505,
|
|
"p50": 1178.5299420007505,
|
|
"p90": 1178.5299420007505,
|
|
"p95": 1178.5299420007505,
|
|
"p99": 1178.5299420007505
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 393216,
|
|
"max_model_len": 394240,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 394240 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 1.3780717330082553,
|
|
"request_throughput": 0.725651630497532,
|
|
"input_token_throughput": 285337.83153771755,
|
|
"output_token_throughput": 0.725651630497532,
|
|
"total_input_tokens": 393216,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 1371.242612003698,
|
|
"p50": 1371.242612003698,
|
|
"p90": 1371.242612003698,
|
|
"p95": 1371.242612003698,
|
|
"p99": 1371.242612003698
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 1371.1869800026761,
|
|
"p50": 1371.1869800026761,
|
|
"p90": 1371.1869800026761,
|
|
"p95": 1371.1869800026761,
|
|
"p99": 1371.1869800026761
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 524288,
|
|
"max_model_len": 525312,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 525312 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 2.587135097011924,
|
|
"request_throughput": 0.3865279401740462,
|
|
"input_token_throughput": 202651.96069797032,
|
|
"output_token_throughput": 0.3865279401740462,
|
|
"total_input_tokens": 524288,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 2582.903844988323,
|
|
"p50": 2582.903844988323,
|
|
"p90": 2582.903844988323,
|
|
"p95": 2582.903844988323,
|
|
"p99": 2582.903844988323
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 2582.850620994577,
|
|
"p50": 2582.850620994577,
|
|
"p90": 2582.850620994577,
|
|
"p95": 2582.850620994577,
|
|
"p99": 2582.850620994577
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 655360,
|
|
"max_model_len": 656384,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 656384 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 2.206327235006029,
|
|
"request_throughput": 0.453241923561383,
|
|
"input_token_throughput": 297036.62702518795,
|
|
"output_token_throughput": 0.453241923561383,
|
|
"total_input_tokens": 655360,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 2203.5920649941545,
|
|
"p50": 2203.5920649941545,
|
|
"p90": 2203.5920649941545,
|
|
"p95": 2203.5920649941545,
|
|
"p99": 2203.5920649941545
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 2203.5506129905116,
|
|
"p50": 2203.5506129905116,
|
|
"p90": 2203.5506129905116,
|
|
"p95": 2203.5506129905116,
|
|
"p99": 2203.5506129905116
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 786432,
|
|
"max_model_len": 787456,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 787456 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 3.8894555929873604,
|
|
"request_throughput": 0.25710539073977023,
|
|
"input_token_throughput": 202195.90665025898,
|
|
"output_token_throughput": 0.25710539073977023,
|
|
"total_input_tokens": 786432,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 3886.5709000092465,
|
|
"p50": 3886.5709000092465,
|
|
"p90": 3886.5709000092465,
|
|
"p95": 3886.5709000092465,
|
|
"p99": 3886.5709000092465
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 3886.5172150108265,
|
|
"p50": 3886.5172150108265,
|
|
"p90": 3886.5172150108265,
|
|
"p95": 3886.5172150108265,
|
|
"p99": 3886.5172150108265
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 917504,
|
|
"max_model_len": 918528,
|
|
"success": true,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 918528 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 3.7180563760048244,
|
|
"request_throughput": 0.26895772921940825,
|
|
"input_token_throughput": 246769.79238972397,
|
|
"output_token_throughput": 0.26895772921940825,
|
|
"total_input_tokens": 917504,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 3715.635304004536,
|
|
"p50": 3715.635304004536,
|
|
"p90": 3715.635304004536,
|
|
"p95": 3715.635304004536,
|
|
"p99": 3715.635304004536
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 3715.6081900029676,
|
|
"p50": 3715.6081900029676,
|
|
"p90": 3715.6081900029676,
|
|
"p95": 3715.6081900029676,
|
|
"p99": 3715.6081900029676
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
},
|
|
{
|
|
"backend": "vllm",
|
|
"target_len": 1048576,
|
|
"max_model_len": 1049600,
|
|
"success": false,
|
|
"server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 1049600 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
|
|
"metrics": null,
|
|
"error": "server failed to start"
|
|
}
|
|
]
|
|
} |