From a56c995091df13250e18e39e042fac4c23273f75 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Wed, 8 Jul 2026 15:33:48 +0000 Subject: [PATCH] results(max-context): complete H200 max context length exploration run 20260708-143534 --- .../results/20260708-143534/report.md | 36 + .../results/20260708-143534/results.json | 783 ++++++++++++++++++ 2 files changed, 819 insertions(+) create mode 100644 experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md create mode 100644 experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json diff --git a/experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md b/experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md new file mode 100644 index 0000000..80a70bd --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md @@ -0,0 +1,36 @@ +# H200 Max Context Length Exploration Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_max_context_length/results/20260708-143534` +- Model: `/data/models/DeepSeek-V4-Flash` +- Hardware: 8x NVIDIA H200 143GB +- TP: 8 +- Output len: 1, num prompts: 1, concurrency: 1 + +## Per-attempt results + +| Backend | Target len | Max model/context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| sglang | 65536 | 66560 | ✅ pass | 521.23 | 521.23 | 521.25 | 521.25 | | +| sglang | 131072 | 132096 | ✅ pass | 668.67 | 668.67 | 668.69 | 668.69 | | +| sglang | 262144 | 263168 | ✅ pass | 1232.63 | 1232.63 | 1232.65 | 1232.65 | | +| sglang | 393216 | 394240 | ✅ pass | 1496.49 | 1496.49 | 1496.53 | 1496.53 | | +| sglang | 524288 | 525312 | ✅ pass | 1764.18 | 1764.18 | 1764.23 | 1764.23 | | +| sglang | 655360 | 656384 | ✅ pass | 2129.19 | 2129.19 | 2129.24 | 2129.24 | | +| sglang | 786432 | 787456 | ✅ pass | 2440.34 | 2440.34 | 2440.35 | 2440.35 | | +| sglang | 917504 | 918528 | ✅ pass | 3149.58 | 3149.58 | 3149.61 | 3149.61 | | +| sglang | 1048576 | 1049600 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | server failed to start | +| vllm | 65536 | 66560 | ✅ pass | 594.47 | 594.47 | 594.53 | 594.53 | | +| vllm | 131072 | 132096 | ✅ pass | 889.46 | 889.46 | 889.51 | 889.51 | | +| vllm | 262144 | 263168 | ✅ pass | 1178.53 | 1178.53 | 1178.56 | 1178.56 | | +| vllm | 393216 | 394240 | ✅ pass | 1371.19 | 1371.19 | 1371.24 | 1371.24 | | +| vllm | 524288 | 525312 | ✅ pass | 2582.85 | 2582.85 | 2582.90 | 2582.90 | | +| vllm | 655360 | 656384 | ✅ pass | 2203.55 | 2203.55 | 2203.59 | 2203.59 | | +| vllm | 786432 | 787456 | ✅ pass | 3886.52 | 3886.52 | 3886.57 | 3886.57 | | +| vllm | 917504 | 918528 | ✅ pass | 3715.61 | 3715.61 | 3715.64 | 3715.64 | | +| vllm | 1048576 | 1049600 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | server failed to start | + +## Maximum supported input length + +- **sglang**: 917504 tokens +- **vllm**: 917504 tokens + diff --git a/experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json b/experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json new file mode 100644 index 0000000..9632761 --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json @@ -0,0 +1,783 @@ +{ + "metadata": { + "experiment": "dsv4_h200_max_context_length", + "run_id": "20260708-143534", + "timestamp": "2026-07-08T14:35:34+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang+vllm", + "engine": "sglang+vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_max_context_length/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "ef65885", + "git_dirty": "dirty", + "description": "H200 max context length exploration for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024 + }, + "scenarios": [ + { + "backend": "sglang", + "target_len": 65536, + "max_model_len": 66560, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 66560 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.5241995020041941, + "request_throughput": 1.9076706409995772, + "input_token_throughput": 125021.10312854829, + "output_token_throughput": 1.9076706409995772, + "total_input_tokens": 65536, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 521.2510440032929, + "p50": 521.2510440032929, + "p90": 521.2510440032929, + "p95": 521.2510440032929, + "p99": 521.2510440032929 + }, + "ttft_ms": { + "mean": 521.2282620050246, + "p50": 521.2282620050246, + "p90": 521.2282620050246, + "p95": 521.2282620050246, + "p99": 521.2282620050246 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 131072, + "max_model_len": 132096, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 132096 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.6736304189980729, + "request_throughput": 1.484493532057763, + "input_token_throughput": 194575.53623387511, + "output_token_throughput": 1.484493532057763, + "total_input_tokens": 131072, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 668.692392995581, + "p50": 668.692392995581, + "p90": 668.692392995581, + "p95": 668.692392995581, + "p99": 668.692392995581 + }, + "ttft_ms": { + "mean": 668.666598998243, + "p50": 668.666598998243, + "p90": 668.666598998243, + "p95": 668.666598998243, + "p99": 668.666598998243 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 262144, + "max_model_len": 263168, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 263168 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.235619597006007, + "request_throughput": 0.8093105697118029, + "input_token_throughput": 212155.90998653087, + "output_token_throughput": 0.8093105697118029, + "total_input_tokens": 262144, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1232.6548010023544, + "p50": 1232.6548010023544, + "p90": 1232.6548010023544, + "p95": 1232.6548010023544, + "p99": 1232.6548010023544 + }, + "ttft_ms": { + "mean": 1232.628379992093, + "p50": 1232.628379992093, + "p90": 1232.628379992093, + "p95": 1232.628379992093, + "p99": 1232.628379992093 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 393216, + "max_model_len": 394240, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 394240 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.4997316709923325, + "request_throughput": 0.6667859453406932, + "input_token_throughput": 262190.90228308603, + "output_token_throughput": 0.6667859453406932, + "total_input_tokens": 393216, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1496.5319500042824, + "p50": 1496.5319500042824, + "p90": 1496.5319500042824, + "p95": 1496.5319500042824, + "p99": 1496.5319500042824 + }, + "ttft_ms": { + "mean": 1496.4891230047215, + "p50": 1496.4891230047215, + "p90": 1496.4891230047215, + "p95": 1496.4891230047215, + "p99": 1496.4891230047215 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 524288, + "max_model_len": 525312, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 525312 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.7669223830016563, + "request_throughput": 0.5659558165204717, + "input_token_throughput": 296723.84313188505, + "output_token_throughput": 0.5659558165204717, + "total_input_tokens": 524288, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1764.228992004064, + "p50": 1764.228992004064, + "p90": 1764.228992004064, + "p95": 1764.228992004064, + "p99": 1764.228992004064 + }, + "ttft_ms": { + "mean": 1764.184692001436, + "p50": 1764.184692001436, + "p90": 1764.184692001436, + "p95": 1764.184692001436, + "p99": 1764.184692001436 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 655360, + "max_model_len": 656384, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 656384 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.1324244450079277, + "request_throughput": 0.4689497920271132, + "input_token_throughput": 307330.93570288894, + "output_token_throughput": 0.4689497920271132, + "total_input_tokens": 655360, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2129.2401870014146, + "p50": 2129.2401870014146, + "p90": 2129.2401870014146, + "p95": 2129.2401870014146, + "p99": 2129.2401870014146 + }, + "ttft_ms": { + "mean": 2129.1923570097424, + "p50": 2129.1923570097424, + "p90": 2129.1923570097424, + "p95": 2129.1923570097424, + "p99": 2129.1923570097424 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 786432, + "max_model_len": 787456, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 787456 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.4429005489946576, + "request_throughput": 0.409349451581865, + "input_token_throughput": 321925.50790642924, + "output_token_throughput": 0.409349451581865, + "total_input_tokens": 786432, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2440.3524239896797, + "p50": 2440.3524239896797, + "p90": 2440.3524239896797, + "p95": 2440.3524239896797, + "p99": 2440.3524239896797 + }, + "ttft_ms": { + "mean": 2440.341288995114, + "p50": 2440.341288995114, + "p90": 2440.341288995114, + "p95": 2440.341288995114, + "p99": 2440.341288995114 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 917504, + "max_model_len": 918528, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 918528 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.152248802012764, + "request_throughput": 0.31723384250681075, + "input_token_throughput": 291063.3194353689, + "output_token_throughput": 0.31723384250681075, + "total_input_tokens": 917504, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 3149.605149999843, + "p50": 3149.605149999843, + "p90": 3149.605149999843, + "p95": 3149.605149999843, + "p99": 3149.605149999843 + }, + "ttft_ms": { + "mean": 3149.579901000834, + "p50": 3149.579901000834, + "p90": 3149.579901000834, + "p95": 3149.579901000834, + "p99": 3149.579901000834 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 1048576, + "max_model_len": 1049600, + "success": false, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 1049600 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": null, + "error": "server failed to start" + }, + { + "backend": "vllm", + "target_len": 65536, + "max_model_len": 66560, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 66560 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.5968579519976629, + "request_throughput": 1.675440524253777, + "input_token_throughput": 109801.67019749554, + "output_token_throughput": 1.675440524253777, + "total_input_tokens": 65536, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 594.5277150021866, + "p50": 594.5277150021866, + "p90": 594.5277150021866, + "p95": 594.5277150021866, + "p99": 594.5277150021866 + }, + "ttft_ms": { + "mean": 594.4739139958983, + "p50": 594.4739139958983, + "p90": 594.4739139958983, + "p95": 594.4739139958983, + "p99": 594.4739139958983 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 131072, + "max_model_len": 132096, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 132096 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.8928074569994351, + "request_throughput": 1.1200623294084255, + "input_token_throughput": 146808.80964022115, + "output_token_throughput": 1.1200623294084255, + "total_input_tokens": 131072, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 889.5100749941776, + "p50": 889.5100749941776, + "p90": 889.5100749941776, + "p95": 889.5100749941776, + "p99": 889.5100749941776 + }, + "ttft_ms": { + "mean": 889.4585100060795, + "p50": 889.4585100060795, + "p90": 889.4585100060795, + "p95": 889.4585100060795, + "p99": 889.4585100060795 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 262144, + "max_model_len": 263168, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 263168 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.1815237270056969, + "request_throughput": 0.8463647213706597, + "input_token_throughput": 221869.4335189902, + "output_token_throughput": 0.8463647213706597, + "total_input_tokens": 262144, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1178.564740010188, + "p50": 1178.564740010188, + "p90": 1178.564740010188, + "p95": 1178.564740010188, + "p99": 1178.564740010188 + }, + "ttft_ms": { + "mean": 1178.5299420007505, + "p50": 1178.5299420007505, + "p90": 1178.5299420007505, + "p95": 1178.5299420007505, + "p99": 1178.5299420007505 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 393216, + "max_model_len": 394240, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 394240 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.3780717330082553, + "request_throughput": 0.725651630497532, + "input_token_throughput": 285337.83153771755, + "output_token_throughput": 0.725651630497532, + "total_input_tokens": 393216, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1371.242612003698, + "p50": 1371.242612003698, + "p90": 1371.242612003698, + "p95": 1371.242612003698, + "p99": 1371.242612003698 + }, + "ttft_ms": { + "mean": 1371.1869800026761, + "p50": 1371.1869800026761, + "p90": 1371.1869800026761, + "p95": 1371.1869800026761, + "p99": 1371.1869800026761 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 524288, + "max_model_len": 525312, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 525312 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.587135097011924, + "request_throughput": 0.3865279401740462, + "input_token_throughput": 202651.96069797032, + "output_token_throughput": 0.3865279401740462, + "total_input_tokens": 524288, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2582.903844988323, + "p50": 2582.903844988323, + "p90": 2582.903844988323, + "p95": 2582.903844988323, + "p99": 2582.903844988323 + }, + "ttft_ms": { + "mean": 2582.850620994577, + "p50": 2582.850620994577, + "p90": 2582.850620994577, + "p95": 2582.850620994577, + "p99": 2582.850620994577 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 655360, + "max_model_len": 656384, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 656384 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.206327235006029, + "request_throughput": 0.453241923561383, + "input_token_throughput": 297036.62702518795, + "output_token_throughput": 0.453241923561383, + "total_input_tokens": 655360, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2203.5920649941545, + "p50": 2203.5920649941545, + "p90": 2203.5920649941545, + "p95": 2203.5920649941545, + "p99": 2203.5920649941545 + }, + "ttft_ms": { + "mean": 2203.5506129905116, + "p50": 2203.5506129905116, + "p90": 2203.5506129905116, + "p95": 2203.5506129905116, + "p99": 2203.5506129905116 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 786432, + "max_model_len": 787456, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 787456 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.8894555929873604, + "request_throughput": 0.25710539073977023, + "input_token_throughput": 202195.90665025898, + "output_token_throughput": 0.25710539073977023, + "total_input_tokens": 786432, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 3886.5709000092465, + "p50": 3886.5709000092465, + "p90": 3886.5709000092465, + "p95": 3886.5709000092465, + "p99": 3886.5709000092465 + }, + "ttft_ms": { + "mean": 3886.5172150108265, + "p50": 3886.5172150108265, + "p90": 3886.5172150108265, + "p95": 3886.5172150108265, + "p99": 3886.5172150108265 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 917504, + "max_model_len": 918528, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 918528 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.7180563760048244, + "request_throughput": 0.26895772921940825, + "input_token_throughput": 246769.79238972397, + "output_token_throughput": 0.26895772921940825, + "total_input_tokens": 917504, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 3715.635304004536, + "p50": 3715.635304004536, + "p90": 3715.635304004536, + "p95": 3715.635304004536, + "p99": 3715.635304004536 + }, + "ttft_ms": { + "mean": 3715.6081900029676, + "p50": 3715.6081900029676, + "p90": 3715.6081900029676, + "p95": 3715.6081900029676, + "p99": 3715.6081900029676 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 1048576, + "max_model_len": 1049600, + "success": false, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 1049600 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": null, + "error": "server failed to start" + } + ] +} \ No newline at end of file