From a56c995091df13250e18e39e042fac4c23273f75 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Wed, 8 Jul 2026 15:33:48 +0000 Subject: [PATCH 01/53] results(max-context): complete H200 max context length exploration run 20260708-143534 --- .../results/20260708-143534/report.md | 36 + .../results/20260708-143534/results.json | 783 ++++++++++++++++++ 2 files changed, 819 insertions(+) create mode 100644 experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md create mode 100644 experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json diff --git a/experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md b/experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md new file mode 100644 index 0000000..80a70bd --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/results/20260708-143534/report.md @@ -0,0 +1,36 @@ +# H200 Max Context Length Exploration Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_max_context_length/results/20260708-143534` +- Model: `/data/models/DeepSeek-V4-Flash` +- Hardware: 8x NVIDIA H200 143GB +- TP: 8 +- Output len: 1, num prompts: 1, concurrency: 1 + +## Per-attempt results + +| Backend | Target len | Max model/context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| sglang | 65536 | 66560 | ✅ pass | 521.23 | 521.23 | 521.25 | 521.25 | | +| sglang | 131072 | 132096 | ✅ pass | 668.67 | 668.67 | 668.69 | 668.69 | | +| sglang | 262144 | 263168 | ✅ pass | 1232.63 | 1232.63 | 1232.65 | 1232.65 | | +| sglang | 393216 | 394240 | ✅ pass | 1496.49 | 1496.49 | 1496.53 | 1496.53 | | +| sglang | 524288 | 525312 | ✅ pass | 1764.18 | 1764.18 | 1764.23 | 1764.23 | | +| sglang | 655360 | 656384 | ✅ pass | 2129.19 | 2129.19 | 2129.24 | 2129.24 | | +| sglang | 786432 | 787456 | ✅ pass | 2440.34 | 2440.34 | 2440.35 | 2440.35 | | +| sglang | 917504 | 918528 | ✅ pass | 3149.58 | 3149.58 | 3149.61 | 3149.61 | | +| sglang | 1048576 | 1049600 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | server failed to start | +| vllm | 65536 | 66560 | ✅ pass | 594.47 | 594.47 | 594.53 | 594.53 | | +| vllm | 131072 | 132096 | ✅ pass | 889.46 | 889.46 | 889.51 | 889.51 | | +| vllm | 262144 | 263168 | ✅ pass | 1178.53 | 1178.53 | 1178.56 | 1178.56 | | +| vllm | 393216 | 394240 | ✅ pass | 1371.19 | 1371.19 | 1371.24 | 1371.24 | | +| vllm | 524288 | 525312 | ✅ pass | 2582.85 | 2582.85 | 2582.90 | 2582.90 | | +| vllm | 655360 | 656384 | ✅ pass | 2203.55 | 2203.55 | 2203.59 | 2203.59 | | +| vllm | 786432 | 787456 | ✅ pass | 3886.52 | 3886.52 | 3886.57 | 3886.57 | | +| vllm | 917504 | 918528 | ✅ pass | 3715.61 | 3715.61 | 3715.64 | 3715.64 | | +| vllm | 1048576 | 1049600 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | server failed to start | + +## Maximum supported input length + +- **sglang**: 917504 tokens +- **vllm**: 917504 tokens + diff --git a/experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json b/experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json new file mode 100644 index 0000000..9632761 --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/results/20260708-143534/results.json @@ -0,0 +1,783 @@ +{ + "metadata": { + "experiment": "dsv4_h200_max_context_length", + "run_id": "20260708-143534", + "timestamp": "2026-07-08T14:35:34+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang+vllm", + "engine": "sglang+vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_max_context_length/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "ef65885", + "git_dirty": "dirty", + "description": "H200 max context length exploration for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024 + }, + "scenarios": [ + { + "backend": "sglang", + "target_len": 65536, + "max_model_len": 66560, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 66560 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.5241995020041941, + "request_throughput": 1.9076706409995772, + "input_token_throughput": 125021.10312854829, + "output_token_throughput": 1.9076706409995772, + "total_input_tokens": 65536, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 521.2510440032929, + "p50": 521.2510440032929, + "p90": 521.2510440032929, + "p95": 521.2510440032929, + "p99": 521.2510440032929 + }, + "ttft_ms": { + "mean": 521.2282620050246, + "p50": 521.2282620050246, + "p90": 521.2282620050246, + "p95": 521.2282620050246, + "p99": 521.2282620050246 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 131072, + "max_model_len": 132096, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 132096 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.6736304189980729, + "request_throughput": 1.484493532057763, + "input_token_throughput": 194575.53623387511, + "output_token_throughput": 1.484493532057763, + "total_input_tokens": 131072, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 668.692392995581, + "p50": 668.692392995581, + "p90": 668.692392995581, + "p95": 668.692392995581, + "p99": 668.692392995581 + }, + "ttft_ms": { + "mean": 668.666598998243, + "p50": 668.666598998243, + "p90": 668.666598998243, + "p95": 668.666598998243, + "p99": 668.666598998243 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 262144, + "max_model_len": 263168, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 263168 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.235619597006007, + "request_throughput": 0.8093105697118029, + "input_token_throughput": 212155.90998653087, + "output_token_throughput": 0.8093105697118029, + "total_input_tokens": 262144, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1232.6548010023544, + "p50": 1232.6548010023544, + "p90": 1232.6548010023544, + "p95": 1232.6548010023544, + "p99": 1232.6548010023544 + }, + "ttft_ms": { + "mean": 1232.628379992093, + "p50": 1232.628379992093, + "p90": 1232.628379992093, + "p95": 1232.628379992093, + "p99": 1232.628379992093 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 393216, + "max_model_len": 394240, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 394240 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.4997316709923325, + "request_throughput": 0.6667859453406932, + "input_token_throughput": 262190.90228308603, + "output_token_throughput": 0.6667859453406932, + "total_input_tokens": 393216, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1496.5319500042824, + "p50": 1496.5319500042824, + "p90": 1496.5319500042824, + "p95": 1496.5319500042824, + "p99": 1496.5319500042824 + }, + "ttft_ms": { + "mean": 1496.4891230047215, + "p50": 1496.4891230047215, + "p90": 1496.4891230047215, + "p95": 1496.4891230047215, + "p99": 1496.4891230047215 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 524288, + "max_model_len": 525312, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 525312 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.7669223830016563, + "request_throughput": 0.5659558165204717, + "input_token_throughput": 296723.84313188505, + "output_token_throughput": 0.5659558165204717, + "total_input_tokens": 524288, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1764.228992004064, + "p50": 1764.228992004064, + "p90": 1764.228992004064, + "p95": 1764.228992004064, + "p99": 1764.228992004064 + }, + "ttft_ms": { + "mean": 1764.184692001436, + "p50": 1764.184692001436, + "p90": 1764.184692001436, + "p95": 1764.184692001436, + "p99": 1764.184692001436 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 655360, + "max_model_len": 656384, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 656384 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.1324244450079277, + "request_throughput": 0.4689497920271132, + "input_token_throughput": 307330.93570288894, + "output_token_throughput": 0.4689497920271132, + "total_input_tokens": 655360, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2129.2401870014146, + "p50": 2129.2401870014146, + "p90": 2129.2401870014146, + "p95": 2129.2401870014146, + "p99": 2129.2401870014146 + }, + "ttft_ms": { + "mean": 2129.1923570097424, + "p50": 2129.1923570097424, + "p90": 2129.1923570097424, + "p95": 2129.1923570097424, + "p99": 2129.1923570097424 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 786432, + "max_model_len": 787456, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 787456 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.4429005489946576, + "request_throughput": 0.409349451581865, + "input_token_throughput": 321925.50790642924, + "output_token_throughput": 0.409349451581865, + "total_input_tokens": 786432, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2440.3524239896797, + "p50": 2440.3524239896797, + "p90": 2440.3524239896797, + "p95": 2440.3524239896797, + "p99": 2440.3524239896797 + }, + "ttft_ms": { + "mean": 2440.341288995114, + "p50": 2440.341288995114, + "p90": 2440.341288995114, + "p95": 2440.341288995114, + "p99": 2440.341288995114 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 917504, + "max_model_len": 918528, + "success": true, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 918528 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.152248802012764, + "request_throughput": 0.31723384250681075, + "input_token_throughput": 291063.3194353689, + "output_token_throughput": 0.31723384250681075, + "total_input_tokens": 917504, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 3149.605149999843, + "p50": 3149.605149999843, + "p90": 3149.605149999843, + "p95": 3149.605149999843, + "p99": 3149.605149999843 + }, + "ttft_ms": { + "mean": 3149.579901000834, + "p50": 3149.579901000834, + "p90": 3149.579901000834, + "p95": 3149.579901000834, + "p99": 3149.579901000834 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 1048576, + "max_model_len": 1049600, + "success": false, + "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 1049600 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011", + "metrics": null, + "error": "server failed to start" + }, + { + "backend": "vllm", + "target_len": 65536, + "max_model_len": 66560, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 66560 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.5968579519976629, + "request_throughput": 1.675440524253777, + "input_token_throughput": 109801.67019749554, + "output_token_throughput": 1.675440524253777, + "total_input_tokens": 65536, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 594.5277150021866, + "p50": 594.5277150021866, + "p90": 594.5277150021866, + "p95": 594.5277150021866, + "p99": 594.5277150021866 + }, + "ttft_ms": { + "mean": 594.4739139958983, + "p50": 594.4739139958983, + "p90": 594.4739139958983, + "p95": 594.4739139958983, + "p99": 594.4739139958983 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 131072, + "max_model_len": 132096, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 132096 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.8928074569994351, + "request_throughput": 1.1200623294084255, + "input_token_throughput": 146808.80964022115, + "output_token_throughput": 1.1200623294084255, + "total_input_tokens": 131072, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 889.5100749941776, + "p50": 889.5100749941776, + "p90": 889.5100749941776, + "p95": 889.5100749941776, + "p99": 889.5100749941776 + }, + "ttft_ms": { + "mean": 889.4585100060795, + "p50": 889.4585100060795, + "p90": 889.4585100060795, + "p95": 889.4585100060795, + "p99": 889.4585100060795 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 262144, + "max_model_len": 263168, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 263168 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.1815237270056969, + "request_throughput": 0.8463647213706597, + "input_token_throughput": 221869.4335189902, + "output_token_throughput": 0.8463647213706597, + "total_input_tokens": 262144, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1178.564740010188, + "p50": 1178.564740010188, + "p90": 1178.564740010188, + "p95": 1178.564740010188, + "p99": 1178.564740010188 + }, + "ttft_ms": { + "mean": 1178.5299420007505, + "p50": 1178.5299420007505, + "p90": 1178.5299420007505, + "p95": 1178.5299420007505, + "p99": 1178.5299420007505 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 393216, + "max_model_len": 394240, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 394240 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 1.3780717330082553, + "request_throughput": 0.725651630497532, + "input_token_throughput": 285337.83153771755, + "output_token_throughput": 0.725651630497532, + "total_input_tokens": 393216, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 1371.242612003698, + "p50": 1371.242612003698, + "p90": 1371.242612003698, + "p95": 1371.242612003698, + "p99": 1371.242612003698 + }, + "ttft_ms": { + "mean": 1371.1869800026761, + "p50": 1371.1869800026761, + "p90": 1371.1869800026761, + "p95": 1371.1869800026761, + "p99": 1371.1869800026761 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 524288, + "max_model_len": 525312, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 525312 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.587135097011924, + "request_throughput": 0.3865279401740462, + "input_token_throughput": 202651.96069797032, + "output_token_throughput": 0.3865279401740462, + "total_input_tokens": 524288, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2582.903844988323, + "p50": 2582.903844988323, + "p90": 2582.903844988323, + "p95": 2582.903844988323, + "p99": 2582.903844988323 + }, + "ttft_ms": { + "mean": 2582.850620994577, + "p50": 2582.850620994577, + "p90": 2582.850620994577, + "p95": 2582.850620994577, + "p99": 2582.850620994577 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 655360, + "max_model_len": 656384, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 656384 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 2.206327235006029, + "request_throughput": 0.453241923561383, + "input_token_throughput": 297036.62702518795, + "output_token_throughput": 0.453241923561383, + "total_input_tokens": 655360, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 2203.5920649941545, + "p50": 2203.5920649941545, + "p90": 2203.5920649941545, + "p95": 2203.5920649941545, + "p99": 2203.5920649941545 + }, + "ttft_ms": { + "mean": 2203.5506129905116, + "p50": 2203.5506129905116, + "p90": 2203.5506129905116, + "p95": 2203.5506129905116, + "p99": 2203.5506129905116 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 786432, + "max_model_len": 787456, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 787456 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.8894555929873604, + "request_throughput": 0.25710539073977023, + "input_token_throughput": 202195.90665025898, + "output_token_throughput": 0.25710539073977023, + "total_input_tokens": 786432, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 3886.5709000092465, + "p50": 3886.5709000092465, + "p90": 3886.5709000092465, + "p95": 3886.5709000092465, + "p99": 3886.5709000092465 + }, + "ttft_ms": { + "mean": 3886.5172150108265, + "p50": 3886.5172150108265, + "p90": 3886.5172150108265, + "p95": 3886.5172150108265, + "p99": 3886.5172150108265 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 917504, + "max_model_len": 918528, + "success": true, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 918528 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.7180563760048244, + "request_throughput": 0.26895772921940825, + "input_token_throughput": 246769.79238972397, + "output_token_throughput": 0.26895772921940825, + "total_input_tokens": 917504, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 3715.635304004536, + "p50": 3715.635304004536, + "p90": 3715.635304004536, + "p95": 3715.635304004536, + "p99": 3715.635304004536 + }, + "ttft_ms": { + "mean": 3715.6081900029676, + "p50": 3715.6081900029676, + "p90": 3715.6081900029676, + "p95": 3715.6081900029676, + "p99": 3715.6081900029676 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "vllm", + "target_len": 1048576, + "max_model_len": 1049600, + "success": false, + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 1049600 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012", + "metrics": null, + "error": "server failed to start" + } + ] +} \ No newline at end of file From 7d2ed7f05d15d6eb552fbb9b1c04c44b1f8a5258 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Wed, 8 Jul 2026 16:13:01 +0000 Subject: [PATCH 02/53] Add vLLM+MTP speculative decoding benchmark vs default on H200 - Experiment: experiments/dsv4_h200_vllm_mtp_vs_default/ - Use official DeepSeek-V4 recipe: --speculative-config '{method: mtp, num_speculative_tokens: 1}' - Reuse default-side results from dsv4_h200_vllm_dspark_vs_default - Add TTFT analysis for DSpark vs default trade-offs --- .../results/20260708-142121/ttft_analysis.md | 85 +++++ .../dsv4_h200_vllm_mtp_vs_default/compare.py | 74 +++++ .../dsv4_h200_vllm_mtp_vs_default/config.env | 32 ++ .../results/20260708-160349/comparison.md | 30 ++ .../results/20260708-160349/mtp/report.md | 19 ++ .../results/20260708-160349/mtp/results.json | 311 ++++++++++++++++++ .../run_bench.sh | 216 ++++++++++++ .../start_mtp.sh | 66 ++++ 8 files changed, 833 insertions(+) create mode 100644 experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/ttft_analysis.md create mode 100755 experiments/dsv4_h200_vllm_mtp_vs_default/compare.py create mode 100644 experiments/dsv4_h200_vllm_mtp_vs_default/config.env create mode 100644 experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/comparison.md create mode 100644 experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/report.md create mode 100644 experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/results.json create mode 100755 experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_mtp_vs_default/start_mtp.sh diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/ttft_analysis.md b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/ttft_analysis.md new file mode 100644 index 0000000..be6e009 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/ttft_analysis.md @@ -0,0 +1,85 @@ +# DSpark TTFT 长于 vLLM baseline 的根因分析 + +## 数据回顾 + +| Scenario | Backend | Conc | Input | Output | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | Req/s | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i512_o256 | vllm-dspark | 1 | 512 | 256 | 86.51 | 185.00 | 188.81 | 2.09 | 2.56 | +| c1_i512_o256 | vllm-default | 1 | 512 | 256 | 49.50 | 68.50 | 71.09 | 6.66 | 0.99 | +| c32_i512_o256 | vllm-dspark | 32 | 512 | 256 | 262.56 | 527.21 | 532.11 | 12.06 | 16.53 | +| c32_i512_o256 | vllm-default | 32 | 512 | 256 | 122.62 | 272.00 | 304.80 | 15.77 | 12.98 | +| c128_i512_o256 | vllm-dspark | 128 | 512 | 256 | 778.36 | 1071.85 | 1075.35 | 28.95 | 23.94 | +| c128_i512_o256 | vllm-default | 128 | 512 | 256 | 617.72 | 767.18 | 770.89 | 18.31 | 29.99 | +| c1_i4000_o512 | vllm-dspark | 1 | 4000 | 512 | 123.58 | 176.29 | 190.52 | 1.66 | 1.73 | +| c1_i4000_o512 | vllm-default | 1 | 4000 | 512 | 131.50 | 191.53 | 268.69 | 6.68 | 0.50 | +| c32_i4000_o512 | vllm-dspark | 32 | 4000 | 512 | 298.42 | 434.24 | 522.34 | 9.68 | 11.49 | +| c32_i4000_o512 | vllm-default | 32 | 4000 | 512 | 382.74 | 681.98 | 684.75 | 16.43 | 5.38 | + +## 核心结论 + +**DSpark 在短输入（i=512）场景下 TTFT 明显长于 vLLM baseline，是投机解码机制本身的正常 trade-off，但在当前配置下 overhead 偏大，有优化空间。** + +## 为什么短输入场景 TTFT 会变差？ + +### 1. 投机解码的本质 overhead + +DSpark 在生成第一个 token 时，除了执行一次目标模型（target model）的 prefill，还要： +- 加载并执行 draft model； +- 让 draft model 连续预测最多 `spec-tokens=5` 个 token； +- 用 target model 对这 5 个 token 做并行验证（verification）； +- 根据验证结果回退或接受。 + +对于 **短输入（512 tokens）**，baseline 的 prefill 本身非常快（P50 仅 44-65ms），draft model 的额外 forward + verification 的固定 overhead 占了 TTFT 的很大比例，导致 TTFT 反而变长。 + +### 2. 长输入场景反而受益 + +对于 **长输入（4000 tokens）**，prefill 是主要瓶颈。DSpark 通过 draft model 投机生成后续 token，能显著减少 target model 的 forward 次数，因此： +- c1_i4000_o512：TTFT 与 baseline 接近（123ms vs 131ms），但 TPOT 从 6.68ms 降到 1.66ms； +- c32_i4000_o512：TTFT 明显优于 baseline（298ms vs 383ms，-22%），TPOT 从 16.43ms 降到 9.68ms。 + +这符合投机解码的设计目标：**用 draft model 的额外计算换取 target model 更少的完整 forward，从而在长序列或高输出长度上获得端到端收益。** + +### 3. 高并发短输入场景被放大 + +在 c128_i512_o256 下： +- DSpark 的 Req/s 低于 baseline（23.94 vs 29.99），说明单位时间内处理的请求更少； +- 同时 DSpark 的 mean TPOT 高于 baseline（28.95ms vs 18.31ms）。 + +这说明在并发较高、输入较短时，DSpark 的 draft-verification 流水线没有跑赢 baseline 的简单 decode，导致队列等待时间增加，进一步放大了 TTFT。 + +## 是代码缺陷还是正常现象？ + +### 属于正常现象的部分 + +1. **短输入下投机解码增加 TTFT 是预期行为**：只要 draft model 不是零成本，第一个 token 必然比 baseline 慢。 +2. **DSpark 的收益主要体现在 TPOT/E2E/吞吐量**：所有场景下 DSpark 的 TPOT 都显著低于 baseline，长输入场景 E2E 也大幅缩短。 + +### 可能存在优化空间的部分 + +1. **`spec-tokens=5` 对短输入可能过大** + - 当前 DSpark 使用 `--spec-tokens 5`。 + - 对于 512 token 的短输入，prefill 后只需要 256 个输出 token，一次 spec 5 个 token 的 verification overhead 相对较高。 + - **建议**：对短输入场景尝试 `spec-tokens=1` 或 `spec-tokens=2`，看是否能降低 TTFT overhead 同时保持 TPOT 收益。 + +2. **DSpark 与 default 使用不同 checkpoint** + - DSpark 模型路径是 `/data/models/DeepSeek-V4-Flash-DSpark`，而 baseline 是 `/data/models/DeepSeek-V4-Flash`。 + - 如果 DSpark checkpoint 的 prefill 路径没有被充分优化（例如 MLA/KV cache 格式差异），可能会额外增加 prefill 时间。 + - **建议**：确认两个 checkpoint 在 prefill 阶段是否使用相同的 kernel/量化路径，排除模型差异本身带来的 TTFT 偏移。 + +3. **draft model 与 target model 的调度/重叠** + - 如果 draft model 的 forward 和 target model 的 verification 没有充分 overlap，TTFT 会进一步恶化。 + - **建议**：通过 vLLM 的 profile/profiling 工具查看 prefill 阶段 draft model 是否串行执行，以及是否可以通过 `--spec-max-model-len` 或调度参数优化。 + +4. **高并发下的 batching 策略** + - c128_i512_o256 下 DSpark 吞吐量低于 baseline，可能是 speculative decoding 在高并发下的 batching 效率下降。 + - **建议**：测试不同 `max-num-seqs` 或 `--max-num-batched-tokens` 配置，观察并发吞吐和 TTFT 的变化。 + +## 建议的下一步 + +1. **参数调优实验**：固定模型和硬件，扫描 `spec-tokens ∈ {1, 2, 3, 5}`，重点观察短输入场景的 TTFT 和 TPOT 变化。 +2. **profiling**：在 c1_i512_o256 和 c32_i512_o256 下，用 vLLM 的 `--profile` 或 Nsight 抓取 prefill 时间线，确认 draft model 的 overhead 占比。 +3. **与 MTP 对比**：当前正在跑的 vLLM+MTP 实验会提供另一组投机解码数据，可以对比 DSpark 和官方 MTP 在 TTFT/TPOT 上的 trade-off 差异。 + +## 总结 + +DSpark 在短输入场景 TTFT 变差**不是代码 bug，而是投机解码机制的正常开销**。但当前 `spec-tokens=5` 的配置在短输入下 overhead 偏大，建议通过调参和 profiling 进一步优化，尤其是在 S1 实时交互层（输入短、延迟敏感）场景。 diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/compare.py b/experiments/dsv4_h200_vllm_mtp_vs_default/compare.py new file mode 100755 index 0000000..05db496 --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/compare.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Generate a side-by-side comparison of vLLM+MTP speculative decoding and vLLM default. + +Usage: + python3 compare.py --mtp --default \ + [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--mtp", type=Path, required=True) + parser.add_argument("--default", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + args = parser.parse_args() + + mtp_data = load_result(args.mtp) + default_data = load_result(args.default) + + by_scenario = defaultdict(dict) + for data in (mtp_data, default_data): + backend = data["metadata"]["engine"] + for s in data.get("scenarios", []): + key = s["name"] + by_scenario[key][backend] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write("# vLLM+MTP speculative decoding vs vLLM default on DeepSeek-V4-Flash (H200, TP=8)\n\n") + f.write("## Summary\n\n") + f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") + f.write("- Hardware: 8x NVIDIA H200 143GB\n") + f.write("- Tensor Parallelism: 8\n") + f.write("- Benchmark client: `sglang.bench_serving --backend vllm`\n") + f.write("- Default: no speculative decoding\n") + f.write("- MTP: `--speculative-config '{\\\"method\\\":\\\"mtp\\\",\\\"num_speculative_tokens\\\":1}'` (official DeepSeek-V4 recipe)\n\n") + + f.write("## Side-by-side results\n\n") + f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) |\n") + f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for scenario_name in sorted(by_scenario.keys()): + for backend in ("vllm-mtp", "vllm-default"): + s = by_scenario[scenario_name].get(backend) + if s is None: + continue + cfg = s["config"] + m = s["metrics"] + f.write( + f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- TTFT mean/P95/P99 are the main focus for verifying MTP's impact on time-to-first-token.\n") + f.write("- Mean TPOT and E2E are included to check whether speculative decoding pays back after first token.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/config.env b/experiments/dsv4_h200_vllm_mtp_vs_default/config.env new file mode 100644 index 0000000..a6ee34f --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/config.env @@ -0,0 +1,32 @@ +# Configuration for vLLM MTP speculative decoding vs vLLM default comparison on H200. +# The default-side results are reused from dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default. + +EXPERIMENT="dsv4_h200_vllm_mtp_vs_default" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +MTP_PORT="${MTP_PORT:-30009}" +DEFAULT_RESULT_ROOT="${DEFAULT_RESULT_ROOT:-/data/user1/yy/experiments/dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default}" + +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" +VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" + +export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" +TP=8 + +MAX_MODEL_LEN=32768 +MAX_NUM_SEQS=256 +SPEC_METHOD="${SPEC_METHOD:-mtp}" +SPEC_TOKENS="${SPEC_TOKENS:-1}" + +# Scenarios must match the ones in the reused default result set. +declare -a SCENARIOS=( + "1 512 256 32" + "32 512 256 128" + "128 512 256 128" + "1 4000 512 32" + "32 4000 512 64" +) + +MTP_START_SCRIPT="${SCRIPT_DIR:-.}/start_mtp.sh" diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/comparison.md b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/comparison.md new file mode 100644 index 0000000..0968e44 --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/comparison.md @@ -0,0 +1,30 @@ +# vLLM+MTP speculative decoding vs vLLM default on DeepSeek-V4-Flash (H200, TP=8) + +## Summary + +- Model: `/data/models/DeepSeek-V4-Flash` +- Hardware: 8x NVIDIA H200 143GB +- Tensor Parallelism: 8 +- Benchmark client: `sglang.bench_serving --backend vllm` +- Default: no speculative decoding +- MTP: `--speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":1}'` (official DeepSeek-V4 recipe) + +## Side-by-side results + +| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | vllm-mtp | 128 | 512 | 256 | 35.03 | 4625.47 | 771.59 | 1110.39 | 1114.20 | 18.16 | 111.99 | 2506.29 | +| c128_i512_o256 | vllm-default | 128 | 512 | 256 | 29.99 | 3959.89 | 617.72 | 767.18 | 770.89 | 18.31 | 65.44 | 2695.91 | +| c1_i4000_o512 | vllm-mtp | 1 | 4000 | 512 | 0.83 | 234.90 | 161.27 | 216.12 | 289.08 | 3.70 | 3.89 | 1197.52 | +| c1_i4000_o512 | vllm-default | 1 | 4000 | 512 | 0.50 | 140.13 | 131.50 | 191.53 | 268.69 | 6.68 | 6.73 | 2008.09 | +| c1_i512_o256 | vllm-mtp | 1 | 512 | 256 | 1.64 | 239.37 | 60.63 | 70.69 | 71.97 | 3.80 | 4.33 | 607.04 | +| c1_i512_o256 | vllm-default | 1 | 512 | 256 | 0.99 | 143.82 | 49.50 | 68.50 | 71.09 | 6.66 | 6.71 | 1010.98 | +| c32_i4000_o512 | vllm-mtp | 32 | 4000 | 512 | 6.32 | 1785.09 | 409.17 | 667.77 | 673.18 | 14.18 | 19.84 | 4431.73 | +| c32_i4000_o512 | vllm-default | 32 | 4000 | 512 | 5.38 | 1518.49 | 382.74 | 681.98 | 684.75 | 16.43 | 25.49 | 4927.71 | +| c32_i512_o256 | vllm-mtp | 32 | 512 | 256 | 12.56 | 1657.98 | 279.91 | 761.67 | 766.29 | 16.73 | 37.56 | 2332.50 | +| c32_i512_o256 | vllm-default | 32 | 512 | 256 | 12.98 | 1713.20 | 122.62 | 272.00 | 304.80 | 15.77 | 23.26 | 2154.65 | + +## Notes + +- TTFT mean/P95/P99 are the main focus for verifying MTP's impact on time-to-first-token. +- Mean TPOT and E2E are included to check whether speculative decoding pays back after first token. diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/report.md b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/report.md new file mode 100644 index 0000000..4f6b775 --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/report.md @@ -0,0 +1,19 @@ +# 8x NVIDIA H200 143GB VLLM Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: VLLM +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | mtp | 128 | 512 | 256 | 3.65 | 128 | 35.03 | 9647.26 | 4625.47 | 14272.73 | 771.59 | 1110.39 | 1114.20 | 18.16 | 32.29 | 111.99 | 2506.29 | 3456.20 | 3560.22 | ✅ | +| c1_i4000_o512 | mtp | 1 | 4000 | 512 | 38.35 | 32 | 0.83 | 1694.86 | 234.90 | 1929.76 | 161.27 | 216.12 | 289.08 | 3.70 | 3.82 | 3.89 | 1197.52 | 2034.69 | 2078.68 | ✅ | +| c1_i512_o256 | mtp | 1 | 512 | 256 | 19.46 | 32 | 1.64 | 419.94 | 239.37 | 659.31 | 60.63 | 70.69 | 71.97 | 3.80 | 4.11 | 4.33 | 607.04 | 1022.42 | 1023.52 | ✅ | +| c32_i4000_o512 | mtp | 32 | 4000 | 512 | 10.12 | 64 | 6.32 | 13280.42 | 1785.09 | 15065.50 | 409.17 | 667.77 | 673.18 | 14.18 | 17.48 | 19.84 | 4431.73 | 8157.51 | 8520.71 | ✅ | +| c32_i512_o256 | mtp | 32 | 512 | 256 | 10.19 | 128 | 12.56 | 3458.01 | 1657.98 | 5115.99 | 279.91 | 761.67 | 766.29 | 16.73 | 26.24 | 37.56 | 2332.50 | 4812.90 | 5073.77 | ✅ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/results.json b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/results.json new file mode 100644 index 0000000..940635a --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/results.json @@ -0,0 +1,311 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_mtp_vs_default_mtp", + "run_id": "20260708-160349", + "timestamp": "2026-07-08T16:03:51+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm-mtp", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "ca98fe0", + "git_dirty": "dirty", + "description": "H200 vLLM+MTP TP=8 benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "max_model_len": 32768, + "max_num_seqs": 256, + "spec_method": "mtp", + "spec_tokens": 1, + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_vllm_mtp_vs_default/start_mtp.sh", + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --speculative-config {\\\"method\\\":\\\"mtp\\\",\\\"num_speculative_tokens\\\":1} --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30009" + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "phase": "mtp", + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 3.6536797620065045, + "request_throughput": 35.03317431676217, + "input_token_throughput": 9647.260377478384, + "output_token_throughput": 4625.473796510006, + "total_token_throughput": 14272.734173988389, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 2506.2933596329913, + "p50": 2625.210165999306, + "p90": 3417.3656311962986, + "p95": 3456.20117305225, + "p99": 3560.217050112697 + }, + "ttft_ms": { + "mean": 771.5936086792681, + "p50": 743.3261279948056, + "p90": 964.7660584974801, + "p95": 1110.3887211014808, + "p99": 1114.1990597829863 + }, + "tpot_ms": { + "mean": 18.15787222271614, + "p50": 13.890511986549402, + "p90": 25.0715809341214, + "p95": 32.29210961239336, + "p99": 111.98906169440306 + }, + "itl_ms": { + "mean": 25.75249892379593, + "p50": 21.063708009023685, + "p90": 26.390794408507645, + "p95": 28.396654246898834, + "p99": 255.2068907531792 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/raw_outputs/vllm_mtp_0708_128_512_256.jsonl" + }, + { + "name": "c1_i4000_o512", + "config": { + "phase": "mtp", + "concurrency": 1, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 38.35236829900532, + "request_throughput": 0.8343682911709505, + "input_token_throughput": 1694.8627394591913, + "output_token_throughput": 234.90074797372165, + "total_token_throughput": 1929.7634874329128, + "total_input_tokens": 65002, + "total_output_tokens": 9009, + "e2e_ms": { + "mean": 1197.5197937804296, + "p50": 1222.8141454979777, + "p90": 1945.1575131970458, + "p95": 2034.6854075956799, + "p99": 2078.678046991845 + }, + "ttft_ms": { + "mean": 161.27064584452455, + "p50": 187.0977380021941, + "p90": 207.2812115002307, + "p95": 216.12448994783335, + "p99": 289.0834562029342 + }, + "tpot_ms": { + "mean": 3.698815812859111, + "p50": 3.6894009238415832, + "p90": 3.761323209835616, + "p95": 3.8155798700943198, + "p99": 3.8925407732209094 + }, + "itl_ms": { + "mean": 7.345683533679442, + "p50": 7.343719502387103, + "p90": 7.546291207836475, + "p95": 7.646059650869574, + "p99": 7.946166486799484 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/raw_outputs/vllm_mtp_0708_1_4000_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "phase": "mtp", + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 32 + }, + "metrics": { + "success": 32, + "failed": 0, + "duration_s": 19.455183718004264, + "request_throughput": 1.644805850401016, + "input_token_throughput": 419.9394936805094, + "output_token_throughput": 239.37065141617282, + "total_token_throughput": 659.3101450966822, + "total_input_tokens": 8170, + "total_output_tokens": 4657, + "e2e_ms": { + "mean": 607.0424630620437, + "p50": 709.1372035065433, + "p90": 974.411873005738, + "p95": 1022.4235178509844, + "p99": 1023.5160801192977 + }, + "ttft_ms": { + "mean": 60.633899186541385, + "p50": 67.31531800323864, + "p90": 70.1838248045533, + "p95": 70.6854243442649, + "p99": 71.96889203012688 + }, + "tpot_ms": { + "mean": 3.7982499249353343, + "p50": 3.7510736315107254, + "p90": 3.9678993170763492, + "p95": 4.107961743017731, + "p99": 4.327418892014148 + }, + "itl_ms": { + "mean": 7.296974936146524, + "p50": 7.262905994139146, + "p90": 7.464885500667151, + "p95": 7.645853242138401, + "p99": 9.469499189435757 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/raw_outputs/vllm_mtp_0708_1_512_256.jsonl" + }, + { + "name": "c32_i4000_o512", + "config": { + "phase": "mtp", + "concurrency": 32, + "input_len": 4000, + "output_len": 512, + "dataset": "random", + "num_prompts": 64 + }, + "metrics": { + "success": 64, + "failed": 0, + "duration_s": 10.122198707002099, + "request_throughput": 6.322736971733974, + "input_token_throughput": 13280.415045301297, + "output_token_throughput": 1785.086474097831, + "total_token_throughput": 15065.501519399128, + "total_input_tokens": 134427, + "total_output_tokens": 18069, + "e2e_ms": { + "mean": 4431.7270394060415, + "p50": 4118.452171998797, + "p90": 7965.46315389569, + "p95": 8157.508349696581, + "p99": 8520.712041212246 + }, + "ttft_ms": { + "mean": 409.17025550083963, + "p50": 412.25154899439076, + "p90": 663.4122133953497, + "p95": 667.7715254438226, + "p99": 673.1776816866477 + }, + "tpot_ms": { + "mean": 14.18404794198284, + "p50": 15.168117225958857, + "p90": 17.233481520907418, + "p95": 17.483692691057605, + "p99": 19.836913968959017 + }, + "itl_ms": { + "mean": 28.557059211652966, + "p50": 13.894067000364885, + "p90": 29.86279199831188, + "p95": 174.40551639592738, + "p99": 181.762090458069 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/raw_outputs/vllm_mtp_0708_32_4000_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "phase": "mtp", + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 10.193132108004647, + "request_throughput": 12.55747484126904, + "input_token_throughput": 3458.0146344144614, + "output_token_throughput": 1657.9791001363028, + "total_token_throughput": 5115.993734550764, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 2332.5013259607204, + "p50": 2112.3746865050634, + "p90": 4070.417974996962, + "p95": 4812.899109590944, + "p99": 5073.772265354783 + }, + "ttft_ms": { + "mean": 279.9133442812263, + "p50": 206.422848496004, + "p90": 663.5394259006716, + "p95": 761.6650494928763, + "p99": 766.2871305434965 + }, + "tpot_ms": { + "mean": 16.7286719523913, + "p50": 16.23081167053897, + "p90": 23.97215435499593, + "p95": 26.244743178375007, + "p99": 37.558756268073765 + }, + "itl_ms": { + "mean": 30.42237174675025, + "p50": 14.531328000884969, + "p90": 61.01081449742196, + "p95": 163.01492899947334, + "p99": 182.57800715073245 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/mtp/raw_outputs/vllm_mtp_0708_32_512_256.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh b/experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh new file mode 100755 index 0000000..7715bb3 --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# vLLM+MTP speculative decoding vs vLLM default comparison on H200. +# Reuses the default-side results from dsv4_h200_vllm_dspark_vs_default. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "spec_method=${SPEC_METHOD}" +log "spec_tokens=${SPEC_TOKENS}" +log "default_result_root=${DEFAULT_RESULT_ROOT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local pid_file="/data/user1/yy/${EXPERIMENT_NAME}_mtp.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping mtp server pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true + sleep 2 +} + +start_server() { + log "starting mtp server with ${MTP_START_SCRIPT}" + bash "${MTP_START_SCRIPT}" >> "${log_dir_global}/mtp.server.outer.log" 2>&1 + + if ! is_server_healthy "$MTP_PORT"; then + log "error: mtp server failed to become healthy" + return 1 + fi + log "mtp server is healthy on port ${MTP_PORT}" +} + +run_warmup() { + log "warming up mtp (input=4000, output=512, num=2)" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ + --backend vllm \ + --port "$MTP_PORT" \ + --input-len 4000 \ + --output-len 512 \ + --num 2 \ + --env-python "${VENV_CLIENT}/bin/python" \ + >> "${log_dir_global}/mtp.warmup.log" 2>&1 + log "warmup for mtp completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$(${VENV_CLIENT}/bin/python -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +run_benchmark() { + local result_root="${RESULT_BASE}/${RUN_ID}/mtp" + local raw_dir="${result_root}/raw_outputs" + local bench_log_dir="${result_root}/logs" + + mkdir -p "$raw_dir" "$bench_log_dir" + + log "===== mtp BENCHMARK START =====" + + stop_server + start_server + run_warmup + + for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + output_file="${raw_dir}/vllm_mtp_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${bench_log_dir}/vllm_mtp_c${concurrency}_i${input_len}_o${output_len}.log" + + if scenario_already_completed "$output_file" "$num_prompts"; then + log "skipping already-completed mtp scenario: c=${concurrency} i=${input_len} o=${output_len}" + continue + fi + + log "running mtp scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$MTP_PORT" \ + --dataset-name random \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "ERROR: mtp scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + } + + log "finished mtp scenario: output=${output_file}" + done + + stop_server + log "===== mtp BENCHMARK DONE =====" +} + +parse_mtp() { + local result_root="${RESULT_BASE}/${RUN_ID}/mtp" + log "parsing mtp results in ${result_root}" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ + >> "${result_root}/logs/parse.log" 2>&1 || { + log "WARNING: parser failed for mtp; see ${result_root}/logs/parse.log" + } +} + +write_mtp_metadata() { + local result_root="${RESULT_BASE}/${RUN_ID}/mtp" + ensure_result_root "$result_root" + local meta_json="${result_root}/results.json" + + local server_args + server_args="vllm serve $MODEL_PATH --trust-remote-code --tensor-parallel-size $TP --kv-cache-dtype fp8 --max-model-len $MAX_MODEL_LEN --max-num-seqs $MAX_NUM_SEQS --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --speculative-config {\\\"method\\\":\\\"mtp\\\",\\\"num_speculative_tokens\\\":$SPEC_TOKENS} --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port $MTP_PORT" + + write_metadata_json \ + "$meta_json" \ + "${EXPERIMENT_NAME}_mtp" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "vllm" \ + "vllm-mtp" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_VLLM" \ + "H200 vLLM+MTP TP=8 benchmark for DeepSeek-V4-Flash" + + jq --arg server_args "$server_args" \ + '.config = { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "max_model_len": 32768, + "max_num_seqs": 256, + "spec_method": "mtp", + "spec_tokens": 1, + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_vllm_mtp_vs_default/start_mtp.sh", + "server_args": $server_args + }' "$meta_json" > "${meta_json}.tmp" && mv "${meta_json}.tmp" "$meta_json" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +stop_server +write_mtp_metadata +run_benchmark +parse_mtp + +log "generating comparison report" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ + --mtp "${RESULT_BASE}/${RUN_ID}/mtp" \ + --default "$DEFAULT_RESULT_ROOT" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/start_mtp.sh b/experiments/dsv4_h200_vllm_mtp_vs_default/start_mtp.sh new file mode 100755 index 0000000..c41fab0 --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/start_mtp.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Start vLLM with MTP speculative decoding for the comparison experiment. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs tmp + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/dsv4_h200_vllm_mtp_vs_default_mtp_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_vllm_mtp_vs_default_mtp.pid" + +rm -f "$PID_FILE" + +echo "=== Starting vLLM+MTP server (TP=$TP, spec_method=$SPEC_METHOD, spec_tokens=$SPEC_TOKENS) ===" +echo "Model: $MODEL_PATH" +echo "Port: $MTP_PORT" +echo "Log: $LOG" + +nohup vllm serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":$SPEC_TOKENS}" \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$MTP_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${MTP_PORT}/health" >/dev/null 2>&1; then + echo "vLLM+MTP server is ready at http://127.0.0.1:${MTP_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM+MTP server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM+MTP server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 From 278653b6a64809fd105255164222ba0fff2e5828 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 01:42:46 +0000 Subject: [PATCH 03/53] Add three-way comparison: MTP vs DSpark vs default --- .../20260708-160349/three_way_comparison.md | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/three_way_comparison.md diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/three_way_comparison.md b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/three_way_comparison.md new file mode 100644 index 0000000..6c1b32a --- /dev/null +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/three_way_comparison.md @@ -0,0 +1,86 @@ +# 投机解码策略对比：vLLM+MTP vs vLLM+DSpark vs vLLM default + +> 数据来自同一批 default baseline（`dsv4_h200_vllm_dspark_vs_default/results/20260708-142121/default`），MTP 与 DSpark 分别与之对比后合并。 + +## 配置差异 + +| 维度 | vLLM default | vLLM+MTP | vLLM+DSpark | +|---|---|---|---| +| 模型 checkpoint | `DeepSeek-V4-Flash` | `DeepSeek-V4-Flash` | `DeepSeek-V4-Flash-DSpark` | +| 运行环境 | `envs/vllm` | `envs/vllm` | `envs/vllm-dspark` | +| 投机方法 | 无 | MTP (`num_speculative_tokens=1`) | DSpark (`spec-tokens=5`) | +| 启动参数 | 无投机 | `--speculative-config '{"method":"mtp","num_speculative_tokens":1}'` | `--spec-method dspark --spec-model --spec-tokens 5` | + +**注意**：DSpark 同时换了模型 checkpoint 和 vLLM 环境，因此不完全是"方法 vs 方法"的公平对比，而是"DSpark 方案整体" vs "官方 MTP 方案"的对比。 + +## 三向结果表 + +| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i512_o256 | vllm-default | 1 | 512 | 256 | 0.99 | 143.82 | 49.50 | 68.50 | 6.66 | 6.71 | 1010.98 | +| c1_i512_o256 | vllm-mtp | 1 | 512 | 256 | 1.64 | 239.37 | 60.63 | 70.69 | 3.80 | 4.33 | 607.04 | +| c1_i512_o256 | vllm-dspark | 1 | 512 | 256 | 2.56 | 372.62 | 86.51 | 185.00 | 2.09 | 3.83 | 389.62 | +| c32_i512_o256 | vllm-default | 32 | 512 | 256 | 12.98 | 1713.20 | 122.62 | 272.00 | 15.77 | 23.26 | 2154.65 | +| c32_i512_o256 | vllm-mtp | 32 | 512 | 256 | 12.56 | 1657.98 | 279.91 | 761.67 | 16.73 | 37.56 | 2332.50 | +| c32_i512_o256 | vllm-dspark | 32 | 512 | 256 | 16.53 | 2182.83 | 262.56 | 527.21 | 12.06 | 28.16 | 1738.79 | +| c128_i512_o256 | vllm-default | 128 | 512 | 256 | 29.99 | 3959.89 | 617.72 | 767.18 | 18.31 | 65.44 | 2695.91 | +| c128_i512_o256 | vllm-mtp | 128 | 512 | 256 | 35.03 | 4625.47 | 771.59 | 1110.39 | 18.16 | 111.99 | 2506.29 | +| c128_i512_o256 | vllm-dspark | 128 | 512 | 256 | 23.94 | 3160.81 | 778.36 | 1071.85 | 28.95 | 78.75 | 3632.34 | +| c1_i4000_o512 | vllm-default | 1 | 4000 | 512 | 0.50 | 140.13 | 131.50 | 191.53 | 6.68 | 6.73 | 2008.09 | +| c1_i4000_o512 | vllm-mtp | 1 | 4000 | 512 | 0.83 | 234.90 | 161.27 | 216.12 | 3.70 | 3.89 | 1197.52 | +| c1_i4000_o512 | vllm-dspark | 1 | 4000 | 512 | 1.73 | 486.20 | 123.58 | 176.29 | 1.66 | 3.56 | 578.26 | +| c32_i4000_o512 | vllm-default | 32 | 4000 | 512 | 5.38 | 1518.49 | 382.74 | 681.98 | 16.43 | 25.49 | 4927.71 | +| c32_i4000_o512 | vllm-mtp | 32 | 4000 | 512 | 6.32 | 1785.09 | 409.17 | 667.77 | 14.18 | 19.84 | 4431.73 | +| c32_i4000_o512 | vllm-dspark | 32 | 4000 | 512 | 11.49 | 3242.88 | 298.42 | 434.24 | 9.68 | 54.60 | 2476.82 | + +## 分场景结论 + +### 短输入（512 tokens） + +| 场景 | 最优 TTFT | 最优 TPOT | 最优吞吐 | 综合推荐 | +|---|---|---|---|---| +| c1_i512_o256 | default | DSpark | DSpark | DSpark（TTFT 可接受，TPOT/E2E 大幅提升） | +| c32_i512_o256 | default | DSpark | DSpark | DSpark（TTFT 与 MTP 接近，TPOT 更好） | +| c128_i512_o256 | default | MTP | MTP | default（如果 TTFT 优先）；MTP（如果吞吐优先） | + +### 长输入（4000 tokens） + +| 场景 | 最优 TTFT | 最优 TPOT | 最优吞吐 | 综合推荐 | +|---|---|---|---|---| +| c1_i4000_o512 | DSpark | DSpark | DSpark | DSpark | +| c32_i4000_o512 | DSpark | DSpark | DSpark | DSpark | + +## 总体判断 + +### DSpark 整体更优，但有前提 + +1. **长输入场景（4000 tokens）DSpark 全胜**：TTFT、TPOT、吞吐均优于 MTP 和 default。 +2. **低并发短输入 DSpark 也更好**：虽然 TTFT 比 default 慢，但 TPOT 和 E2E 提升明显，吞吐更高。 +3. **高并发短输入（c128_i512_o256）DSpark 反而不如 MTP**： + - DSpark 的 TPOT（28.95ms）显著差于 MTP（18.16ms）和 default（18.31ms） + - DSpark 吞吐（23.94 req/s）也低于 default（29.99）和 MTP（35.03） + - 这说明 DSpark 的 `spec-tokens=5` 在高并发短输入下会造成较重的 verification/调度开销，整体效率下降。 + +### MTP 的定位 + +- **优势**：使用官方 recipe，部署简单，不需要换模型 checkpoint 和 vLLM 环境；在高并发短输入下吞吐表现最好。 +- **劣势**：长输入和低并发下收益不如 DSpark；在 c32_i512_o256 这类场景中 TTFT overhead 比 DSpark 还大（P95 761ms vs 527ms）。 + +### default 的定位 + +- 在 TTFT 敏感且输入短的场景（尤其是高并发）下，default 仍然是最稳的选择。 + +## 推荐策略 + +| 应用场景 | 推荐方案 | +|---|---| +| 长输入 / 高输出（如文档总结、代码生成） | **DSpark** | +| 低并发交互（如单用户 Chat） | **DSpark**（如果可接受略高 TTFT） | +| 高并发短输入 API（如 S0/S1 轻量层） | **default** 或 **MTP** | +| 不想换模型 checkpoint，又想尝试投机解码 | **MTP** | + +## 后续可验证的优化点 + +1. **给 DSpark 调小 `spec-tokens`**：在短输入高并发场景测试 `spec-tokens=1/2/3`，可能改善 c128_i512_o256 的 TPOT 和吞吐。 +2. **给 MTP 调大 `num_speculative_tokens`**：当前是 1，尝试 2-4，看能否在长输入下接近 DSpark 的收益。 +3. **统一模型 checkpoint 对比**：如果可能，用同一个模型分别跑 DSpark 方法和 MTP 方法，排除模型差异带来的偏差。 From 7e614fb702fc4a8077e7c12c997dde0d7648331e Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 02:36:57 +0000 Subject: [PATCH 04/53] Add DSpark backend to long-context matrix experiment - Add start_dspark.sh with --spec-method dspark --spec-tokens 5 - Add BACKENDS env var to run only selected backends - Fix client/backend mapping for DSpark (bench client uses vllm, parser uses vllm) - Add three-way compare.py for sglang/vllm/dspark - Reuse existing sglang/vllm results from 20260708-100016 - Add DSpark long-context results and comparison --- .../dsv4_h200_long_context_matrix/compare.py | 100 ++ .../dsv4_h200_long_context_matrix/config.env | 7 + .../results/20260709-020227/comparison.md | 83 ++ .../results/20260709-020227/dspark/report.md | 35 + .../20260709-020227/dspark/results.json | 1219 +++++++++++++++++ .../run_bench.sh | 110 +- .../start_dspark.sh | 72 + 7 files changed, 1595 insertions(+), 31 deletions(-) create mode 100755 experiments/dsv4_h200_long_context_matrix/compare.py create mode 100644 experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md create mode 100644 experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md create mode 100644 experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json create mode 100755 experiments/dsv4_h200_long_context_matrix/start_dspark.sh diff --git a/experiments/dsv4_h200_long_context_matrix/compare.py b/experiments/dsv4_h200_long_context_matrix/compare.py new file mode 100755 index 0000000..9445c3d --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/compare.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Generate a side-by-side comparison of SGLang, vLLM, and DSpark results. + +Usage: + python3 compare.py --sglang --vllm --dspark \ + [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "✅" + if ttft_ok or tpot_ok: + return "⚠️" + return "❌" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sglang", type=Path, required=True) + parser.add_argument("--vllm", type=Path, required=True) + parser.add_argument("--dspark", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + sglang_data = load_result(args.sglang) + vllm_data = load_result(args.vllm) + dspark_data = load_result(args.dspark) + + model = sglang_data.get("metadata", {}).get("model", "unknown") + hardware = sglang_data.get("metadata", {}).get("hardware", "unknown") + + # Map engine names to display labels. + engine_to_label = { + "sglang": "sglang", + "vllm": "vllm", + "vllm-dspark": "dspark", + } + + by_scenario = defaultdict(dict) + for data in (sglang_data, vllm_data, dspark_data): + engine = data["metadata"]["engine"] + label = engine_to_label.get(engine, engine) + for s in data.get("scenarios", []): + key = s["name"] + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang vs vLLM vs DSpark long-context matrix ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write("- DSpark flags: `--spec-method dspark --spec-model --spec-tokens 5`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + f.write("## Side-by-side results\n\n") + f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") + f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for scenario_name in sorted(by_scenario.keys()): + for backend in ("sglang", "vllm", "dspark"): + s = by_scenario[scenario_name].get(backend) + if s is None: + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_long_context_matrix/config.env b/experiments/dsv4_h200_long_context_matrix/config.env index b82fd47..fcddcf5 100644 --- a/experiments/dsv4_h200_long_context_matrix/config.env +++ b/experiments/dsv4_h200_long_context_matrix/config.env @@ -10,9 +10,15 @@ SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30006}" VLLM_PORT="${VLLM_PORT:-30005}" +DSPARK_PORT="${DSPARK_PORT:-30013}" VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" +VENV_VLLM_DSPARK="${VENV_VLLM_DSPARK:-/data/user1/yy/envs/vllm-dspark}" + +# DSpark uses a separate model checkpoint and speculative decoding. +DSPARK_MODEL_NAME="DeepSeek-V4-Flash-DSpark" +DSPARK_MODEL_PATH="/data/models/DeepSeek-V4-Flash-DSpark" export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" TP=8 @@ -60,3 +66,4 @@ VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh" VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" +DSPARK_START_SCRIPT="${SCRIPT_DIR:-.}/start_dspark.sh" diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md new file mode 100644 index 0000000..5a4b0a1 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md @@ -0,0 +1,83 @@ +# SGLang vs vLLM vs DSpark long-context matrix (8x NVIDIA H200 143GB) + +## Summary + +- Model: `/data/models/DeepSeek-V4-Flash` +- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark` +- Hardware: 8x NVIDIA H200 143GB +- Benchmark client: `sglang.bench_serving` +- DSpark flags: `--spec-method dspark --spec-model --spec-tokens 5` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +## Side-by-side results + +| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c10_i131072_o1024 | sglang | 10 | 131072 | 1024 | 0.68 | 319.74 | 4147.51 | 7409.82 | 25.69 | 49.89 | 58.92 | 26432.66 | ⚠️ | +| c10_i131072_o1024 | vllm | 10 | 131072 | 1024 | 0.39 | 183.08 | 6596.15 | 6603.12 | 47.10 | 77.53 | 84.06 | 48426.64 | ⚠️ | +| c10_i131072_o1024 | dspark | 10 | 131072 | 1024 | 0.66 | 310.66 | 11615.81 | 11676.24 | 24.58 | 46.24 | 46.56 | 29271.39 | ⚠️ | +| c10_i131072_o256 | sglang | 10 | 131072 | 256 | 0.34 | 42.70 | 26046.62 | 26064.38 | 1374.50 | 1810.56 | 19553.62 | 58326.41 | ❌ | +| c10_i131072_o256 | vllm | 10 | 131072 | 256 | 0.38 | 48.09 | 23250.03 | 24238.33 | 150.22 | 299.91 | 301.72 | 51439.08 | ❌ | +| c10_i131072_o256 | dspark | 10 | 131072 | 256 | 0.40 | 51.33 | 24693.20 | 27039.11 | 59.88 | 78.22 | 316.95 | 36603.69 | ❌ | +| c10_i131072_o4096 | sglang | 10 | 131072 | 4096 | 0.27 | 517.96 | 6177.42 | 6920.28 | 14.21 | 19.92 | 25.87 | 54286.63 | ⚠️ | +| c10_i131072_o4096 | vllm | 10 | 131072 | 4096 | 0.23 | 445.81 | 2096.29 | 2168.81 | 18.57 | 21.47 | 21.61 | 67041.13 | ✅ | +| c10_i131072_o4096 | dspark | 10 | 131072 | 4096 | 1.05 | 1998.23 | 1543.09 | 1546.01 | 3.34 | 4.81 | 4.84 | 14731.59 | ✅ | +| c10_i262144_o1024 | sglang | 10 | 262144 | 1024 | 0.16 | 76.30 | 39446.96 | 43965.67 | 108.42 | 202.43 | 202.90 | 120180.87 | ❌ | +| c10_i262144_o1024 | vllm | 10 | 262144 | 1024 | 0.18 | 84.63 | 34130.22 | 38144.40 | 98.87 | 142.57 | 266.08 | 108658.89 | ❌ | +| c10_i262144_o1024 | dspark | 10 | 262144 | 1024 | 0.20 | 96.79 | 39321.30 | 40783.01 | 48.19 | 67.60 | 68.13 | 91123.07 | ⚠️ | +| c10_i262144_o256 | sglang | 10 | 262144 | 256 | 0.15 | 18.70 | 58074.31 | 62652.59 | 2953.70 | 3625.59 | 41185.32 | 134628.33 | ❌ | +| c10_i262144_o256 | vllm | 10 | 262144 | 256 | 0.18 | 22.76 | 50632.44 | 51883.02 | 229.20 | 339.27 | 344.95 | 110216.92 | ❌ | +| c10_i262144_o256 | dspark | 10 | 262144 | 256 | 0.18 | 23.12 | 64654.53 | 67877.62 | 52.05 | 73.84 | 151.28 | 80196.26 | ❌ | +| c10_i262144_o4096 | sglang | 10 | 262144 | 4096 | 0.14 | 272.46 | 18088.12 | 22579.09 | 28.00 | 43.87 | 45.07 | 120183.75 | ⚠️ | +| c10_i262144_o4096 | vllm | 10 | 262144 | 4096 | 0.11 | 215.86 | 34100.32 | 38022.94 | 37.18 | 56.07 | 62.00 | 157944.21 | ⚠️ | +| c10_i262144_o4096 | dspark | 10 | 262144 | 4096 | 0.18 | 349.94 | 33426.83 | 37512.71 | 20.67 | 35.94 | 44.44 | 103440.52 | ⚠️ | +| c25_i65536_o1024 | sglang | 25 | 65536 | 1024 | 0.67 | 346.70 | 19996.43 | 24478.11 | 149.27 | 395.00 | 2037.09 | 69586.94 | ❌ | +| c25_i65536_o1024 | vllm | 25 | 65536 | 1024 | 0.64 | 332.61 | 24861.20 | 28942.75 | 72.36 | 196.24 | 271.32 | 72689.84 | ❌ | +| c25_i65536_o1024 | dspark | 25 | 65536 | 1024 | 0.80 | 412.74 | 26078.12 | 29135.60 | 42.82 | 56.19 | 133.89 | 60995.55 | ⚠️ | +| c25_i65536_o256 | sglang | 25 | 65536 | 256 | 0.71 | 84.45 | 27219.06 | 31612.70 | 292.84 | 801.18 | 938.43 | 68854.26 | ❌ | +| c25_i65536_o256 | vllm | 25 | 65536 | 256 | 0.81 | 95.60 | 24902.72 | 28974.45 | 180.78 | 273.59 | 278.16 | 60781.45 | ❌ | +| c25_i65536_o256 | dspark | 25 | 65536 | 256 | 0.84 | 98.79 | 27635.11 | 29212.48 | 51.68 | 65.52 | 134.94 | 39437.04 | ❌ | +| c25_i65536_o4096 | sglang | 25 | 65536 | 4096 | 0.41 | 950.01 | 18559.58 | 23249.71 | 23.78 | 42.50 | 82.84 | 96670.12 | ⚠️ | +| c25_i65536_o4096 | vllm | 25 | 65536 | 4096 | 0.32 | 741.71 | 24847.97 | 28840.96 | 28.80 | 46.66 | 63.40 | 129654.04 | ⚠️ | +| c25_i65536_o4096 | dspark | 25 | 65536 | 4096 | 0.61 | 1409.20 | 25048.60 | 29143.30 | 14.72 | 30.57 | 42.07 | 73496.43 | ⚠️ | +| c2_i131072_o1024 | sglang | 2 | 131072 | 1024 | 0.60 | 136.10 | 479.99 | 482.72 | 8.56 | 9.72 | 9.79 | 5480.98 | ✅ | +| c2_i131072_o1024 | vllm | 2 | 131072 | 1024 | 0.70 | 158.36 | 676.14 | 695.90 | 6.49 | 8.20 | 8.34 | 4803.24 | ✅ | +| c2_i131072_o1024 | dspark | 2 | 131072 | 1024 | 2.29 | 514.96 | 679.81 | 691.56 | 1.10 | 1.60 | 1.63 | 1450.74 | ✅ | +| c2_i131072_o256 | sglang | 2 | 131072 | 256 | 1.43 | 139.48 | 589.14 | 605.33 | 8.29 | 8.96 | 9.05 | 1750.12 | ✅ | +| c2_i131072_o256 | vllm | 2 | 131072 | 256 | 1.51 | 147.12 | 684.63 | 700.28 | 7.08 | 9.27 | 9.45 | 1556.46 | ✅ | +| c2_i131072_o256 | dspark | 2 | 131072 | 256 | 3.10 | 301.11 | 617.17 | 634.08 | 1.00 | 1.41 | 1.42 | 780.85 | ✅ | +| c2_i131072_o4096 | sglang | 2 | 131072 | 4096 | 0.11 | 244.38 | 584.43 | 599.22 | 7.53 | 7.61 | 7.62 | 28360.35 | ✅ | +| c2_i131072_o4096 | vllm | 2 | 131072 | 4096 | 0.12 | 262.22 | 671.34 | 688.15 | 7.09 | 7.17 | 7.18 | 26496.04 | ✅ | +| c2_i131072_o4096 | dspark | 2 | 131072 | 4096 | 0.47 | 1076.96 | 540.43 | 541.94 | 1.76 | 2.03 | 2.08 | 6313.36 | ✅ | +| c2_i262144_o1024 | sglang | 2 | 262144 | 1024 | 0.51 | 114.94 | 1007.07 | 1024.10 | 8.36 | 9.03 | 9.06 | 6240.65 | ✅ | +| c2_i262144_o1024 | vllm | 2 | 262144 | 1024 | 0.19 | 43.14 | 8722.76 | 9009.70 | 20.17 | 52.24 | 57.91 | 13826.13 | ⚠️ | +| c2_i262144_o1024 | dspark | 2 | 262144 | 1024 | 0.25 | 55.65 | 10466.90 | 10984.16 | 14.36 | 43.30 | 48.52 | 15361.85 | ⚠️ | +| c2_i262144_o256 | sglang | 2 | 262144 | 256 | 0.29 | 28.44 | 9538.91 | 9980.78 | 14.10 | 26.98 | 29.16 | 11941.34 | ⚠️ | +| c2_i262144_o256 | vllm | 2 | 262144 | 256 | 0.23 | 22.74 | 8625.18 | 8952.31 | 10.18 | 23.01 | 25.44 | 10378.60 | ⚠️ | +| c2_i262144_o256 | dspark | 2 | 262144 | 256 | 0.26 | 25.25 | 10459.44 | 10978.82 | 17.58 | 45.28 | 48.94 | 14620.43 | ⚠️ | +| c2_i262144_o4096 | sglang | 2 | 262144 | 4096 | 0.08 | 189.78 | 9453.57 | 9896.38 | 7.82 | 8.48 | 8.60 | 38723.14 | ⚠️ | +| c2_i262144_o4096 | vllm | 2 | 262144 | 4096 | 0.08 | 180.76 | 8759.63 | 9062.80 | 8.63 | 11.53 | 12.01 | 35881.75 | ⚠️ | +| c2_i262144_o4096 | dspark | 2 | 262144 | 4096 | 0.17 | 384.42 | 8709.00 | 8987.60 | 3.23 | 6.44 | 6.99 | 15113.78 | ⚠️ | +| c5_i131072_o1024 | sglang | 5 | 131072 | 1024 | 0.87 | 321.54 | 1231.23 | 1298.36 | 9.03 | 10.47 | 10.71 | 7298.25 | ✅ | +| c5_i131072_o1024 | vllm | 5 | 131072 | 1024 | 0.90 | 331.65 | 856.18 | 917.80 | 8.93 | 10.48 | 10.62 | 6989.27 | ✅ | +| c5_i131072_o1024 | dspark | 5 | 131072 | 1024 | 2.75 | 1019.06 | 1080.05 | 1176.83 | 2.53 | 3.98 | 4.18 | 2383.07 | ✅ | +| c5_i131072_o256 | sglang | 5 | 131072 | 256 | 1.05 | 146.97 | 3270.23 | 4685.65 | 19.91 | 40.81 | 46.10 | 7815.89 | ⚠️ | +| c5_i131072_o256 | vllm | 5 | 131072 | 256 | 0.39 | 54.07 | 9384.23 | 9677.91 | 51.54 | 107.16 | 117.25 | 19860.69 | ❌ | +| c5_i131072_o256 | dspark | 5 | 131072 | 256 | 0.44 | 60.98 | 10128.32 | 10501.28 | 37.60 | 60.86 | 61.78 | 17256.59 | ⚠️ | +| c5_i131072_o4096 | sglang | 5 | 131072 | 4096 | 0.20 | 378.76 | 4667.56 | 6275.51 | 11.98 | 22.07 | 23.60 | 31176.81 | ⚠️ | +| c5_i131072_o4096 | vllm | 5 | 131072 | 4096 | 0.23 | 447.85 | 1054.01 | 1104.66 | 8.48 | 9.13 | 9.14 | 28818.20 | ✅ | +| c5_i131072_o4096 | dspark | 5 | 131072 | 4096 | 0.70 | 1335.09 | 1156.91 | 1249.38 | 3.20 | 4.19 | 4.21 | 11315.66 | ✅ | +| c5_i262144_o1024 | sglang | 5 | 262144 | 1024 | 0.15 | 56.42 | 20522.02 | 25944.29 | 54.11 | 118.98 | 124.02 | 53444.22 | ❌ | +| c5_i262144_o1024 | vllm | 5 | 262144 | 1024 | 0.16 | 60.32 | 22564.89 | 26639.92 | 47.49 | 100.69 | 109.86 | 53790.59 | ⚠️ | +| c5_i262144_o1024 | dspark | 5 | 262144 | 1024 | 0.19 | 69.32 | 22995.12 | 26781.97 | 38.56 | 57.19 | 57.43 | 40847.11 | ⚠️ | +| c5_i262144_o256 | sglang | 5 | 262144 | 256 | 0.16 | 21.94 | 25875.40 | 30596.95 | 97.05 | 231.75 | 252.98 | 43756.53 | ❌ | +| c5_i262144_o256 | vllm | 5 | 262144 | 256 | 0.19 | 25.85 | 21986.99 | 26381.86 | 123.42 | 202.78 | 215.56 | 45761.90 | ❌ | +| c5_i262144_o256 | dspark | 5 | 262144 | 256 | 0.19 | 26.70 | 29425.38 | 30576.66 | 41.74 | 60.42 | 62.17 | 37097.84 | ⚠️ | +| c5_i262144_o4096 | sglang | 5 | 262144 | 4096 | 0.11 | 209.50 | 16539.50 | 20323.40 | 25.05 | 56.69 | 60.22 | 60383.57 | ⚠️ | +| c5_i262144_o4096 | vllm | 5 | 262144 | 4096 | 0.11 | 205.12 | 22171.78 | 26533.69 | 25.69 | 63.32 | 68.33 | 64086.27 | ⚠️ | +| c5_i262144_o4096 | dspark | 5 | 262144 | 4096 | 0.16 | 299.79 | 22250.95 | 26667.05 | 17.46 | 45.41 | 49.74 | 51820.15 | ⚠️ | + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out. diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md new file mode 100644 index 0000000..fdd3345 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md @@ -0,0 +1,35 @@ +# 8x NVIDIA H200 143GB VLLM Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark` +- Model: `/data/models/DeepSeek-V4-Flash-DSpark` +- Backend: VLLM +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c10_i131072_o1024 | dspark | 10 | 131072 | 1024 | 30.45 | 20 | 0.66 | 48423.00 | 310.66 | 48733.65 | 4033.86 | 11615.81 | 11676.24 | 24.58 | 46.24 | 46.56 | 14685.68 | 28877.70 | 29271.39 | ⚠️ | +| c10_i131072_o256 | dspark | 10 | 131072 | 256 | 49.62 | 20 | 0.40 | 29712.25 | 51.33 | 29763.58 | 16856.00 | 24693.20 | 27039.11 | 59.88 | 78.22 | 316.95 | 22010.51 | 32508.36 | 36603.69 | ❌ | +| c10_i131072_o4096 | dspark | 10 | 131072 | 4096 | 19.08 | 20 | 1.05 | 77265.07 | 1998.23 | 79263.30 | 727.34 | 1543.09 | 1546.01 | 3.34 | 4.81 | 4.84 | 7316.24 | 12401.94 | 14731.59 | ✅ | +| c2_i131072_o1024 | dspark | 2 | 131072 | 1024 | 1.75 | 4 | 2.29 | 138490.59 | 514.96 | 139005.55 | 447.72 | 679.81 | 691.56 | 1.10 | 1.60 | 1.63 | 794.56 | 1362.81 | 1450.74 | ✅ | +| c2_i131072_o256 | dspark | 2 | 131072 | 256 | 1.29 | 4 | 3.10 | 187564.60 | 301.11 | 187865.71 | 419.01 | 617.17 | 634.08 | 1.00 | 1.41 | 1.42 | 536.04 | 747.52 | 780.85 | ✅ | +| c2_i131072_o4096 | dspark | 2 | 131072 | 4096 | 8.44 | 4 | 0.47 | 28698.90 | 1076.96 | 29775.86 | 325.51 | 540.43 | 541.94 | 1.76 | 2.03 | 2.08 | 4193.26 | 6216.99 | 6313.36 | ✅ | +| c5_i131072_o1024 | dspark | 5 | 131072 | 1024 | 3.63 | 10 | 2.75 | 203409.39 | 1019.06 | 204428.44 | 595.29 | 1080.05 | 1176.83 | 2.53 | 3.98 | 4.18 | 1507.07 | 2347.99 | 2383.07 | ✅ | +| c5_i131072_o256 | dspark | 5 | 131072 | 256 | 22.89 | 10 | 0.44 | 32258.45 | 60.98 | 32319.42 | 5806.57 | 10128.32 | 10501.28 | 37.60 | 60.86 | 61.78 | 11223.11 | 16310.02 | 17256.59 | ⚠️ | +| c5_i131072_o4096 | dspark | 5 | 131072 | 4096 | 14.28 | 10 | 0.70 | 51732.19 | 1335.09 | 53067.28 | 560.62 | 1156.91 | 1249.38 | 3.20 | 4.19 | 4.21 | 6029.71 | 10552.87 | 11315.66 | ✅ | +| c10_i262144_o1024 | dspark | 10 | 262144 | 1024 | 97.73 | 20 | 0.20 | 28498.71 | 96.79 | 28595.50 | 24591.83 | 39321.30 | 40783.01 | 48.19 | 67.60 | 68.13 | 45387.50 | 70254.49 | 91123.07 | ⚠️ | +| c10_i262144_o256 | dspark | 10 | 262144 | 256 | 110.17 | 20 | 0.18 | 25281.12 | 23.12 | 25304.24 | 42248.63 | 64654.53 | 67877.62 | 52.05 | 73.84 | 151.28 | 48223.85 | 78728.49 | 80196.26 | ❌ | +| c10_i262144_o4096 | dspark | 10 | 262144 | 4096 | 108.97 | 20 | 0.18 | 25559.57 | 349.94 | 25909.50 | 13472.46 | 33426.83 | 37512.71 | 20.67 | 35.94 | 44.44 | 52210.57 | 100105.13 | 103440.52 | ⚠️ | +| c2_i262144_o1024 | dspark | 2 | 262144 | 1024 | 16.19 | 4 | 0.25 | 31154.91 | 55.65 | 31210.55 | 5853.53 | 10466.90 | 10984.16 | 14.36 | 43.30 | 48.52 | 7871.57 | 14740.72 | 15361.85 | ⚠️ | +| c2_i262144_o256 | dspark | 2 | 262144 | 256 | 15.41 | 4 | 0.26 | 32738.24 | 25.25 | 32763.49 | 5850.43 | 10459.44 | 10978.82 | 17.58 | 45.28 | 48.94 | 7672.82 | 14088.43 | 14620.43 | ⚠️ | +| c2_i262144_o4096 | dspark | 2 | 262144 | 4096 | 23.65 | 4 | 0.17 | 21326.49 | 384.42 | 21710.90 | 5487.63 | 8709.00 | 8987.60 | 3.23 | 6.44 | 6.99 | 11596.58 | 14994.53 | 15113.78 | ⚠️ | +| c5_i262144_o1024 | dspark | 5 | 262144 | 1024 | 53.37 | 10 | 0.19 | 26116.34 | 69.32 | 26185.66 | 13406.14 | 22995.12 | 26781.97 | 38.56 | 57.19 | 57.43 | 25295.54 | 38376.69 | 40847.11 | ⚠️ | +| c5_i262144_o256 | dspark | 5 | 262144 | 256 | 52.29 | 10 | 0.19 | 26655.70 | 26.70 | 26682.40 | 17996.01 | 29425.38 | 30576.66 | 41.74 | 60.42 | 62.17 | 23507.98 | 35234.36 | 37097.84 | ⚠️ | +| c5_i262144_o4096 | dspark | 5 | 262144 | 4096 | 63.58 | 10 | 0.16 | 21924.13 | 299.79 | 22223.92 | 10590.48 | 22250.95 | 26667.05 | 17.46 | 45.41 | 49.74 | 30807.30 | 48506.31 | 51820.15 | ⚠️ | +| c25_i65536_o1024 | dspark | 25 | 65536 | 1024 | 62.71 | 50 | 0.80 | 29366.67 | 412.74 | 29779.41 | 11692.07 | 26078.12 | 29135.60 | 42.82 | 56.19 | 133.89 | 29981.59 | 57554.65 | 60995.55 | ⚠️ | +| c25_i65536_o256 | dspark | 25 | 65536 | 256 | 59.86 | 50 | 0.84 | 30764.36 | 98.79 | 30863.14 | 19438.04 | 27635.11 | 29212.48 | 51.68 | 65.52 | 134.94 | 24957.48 | 37092.86 | 39437.04 | ❌ | +| c25_i65536_o4096 | dspark | 25 | 65536 | 4096 | 82.31 | 50 | 0.61 | 22371.88 | 1409.20 | 23781.09 | 8084.05 | 25048.60 | 29143.30 | 14.72 | 30.57 | 42.07 | 38164.69 | 72245.56 | 73496.43 | ⚠️ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json new file mode 100644 index 0000000..d17c50d --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json @@ -0,0 +1,1219 @@ +{ + "metadata": { + "experiment": "dsv4_h200_long_context_matrix_dspark", + "run_id": "20260709-020227", + "timestamp": "2026-07-09T02:02:29+00:00", + "model": "/data/models/DeepSeek-V4-Flash-DSpark", + "backend": "dspark", + "engine": "vllm-dspark", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_long_context_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm-dspark", + "git_commit": "0aeabad", + "git_dirty": "dirty", + "description": "H200 long-context matrix dspark TP=8 for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "dspark", + "server_start_script": "experiments/dsv4_h200_long_context_matrix/start_dspark.sh", + "groups": [ + { + "label": "64k", + "max_model_len": 80000, + "max_slots": 25 + }, + { + "label": "128k", + "max_model_len": 140000, + "max_slots": 10 + }, + { + "label": "256k", + "max_model_len": 270000, + "max_slots": 10 + } + ] + }, + "scenarios": [ + { + "name": "c10_i131072_o1024", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 30.44836392698926, + "request_throughput": 0.6568497423361428, + "input_token_throughput": 48422.99584750756, + "output_token_throughput": 310.6570856378787, + "total_token_throughput": 48733.652933145444, + "total_input_tokens": 1474401, + "total_output_tokens": 9459, + "e2e_ms": { + "mean": 14685.683184953814, + "p50": 14254.763487013406, + "p90": 27765.363381317, + "p95": 28877.701320237247, + "p99": 29271.392024847442 + }, + "ttft_ms": { + "mean": 4033.855032298016, + "p50": 1451.7911804869073, + "p90": 10187.548868515298, + "p95": 11615.812585884123, + "p99": 11676.239348381641 + }, + "tpot_ms": { + "mean": 24.581426407367566, + "p50": 26.691134347746317, + "p90": 41.43077303036502, + "p95": 46.23705537957289, + "p99": 46.55721496167711 + }, + "itl_ms": { + "mean": 135.43289032106352, + "p50": 21.5837610012386, + "p90": 320.3089441929478, + "p95": 337.2771248104982, + "p99": 389.6573220437858 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_10_131072_1024.jsonl" + }, + { + "name": "c10_i131072_o256", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 49.62265793100232, + "request_throughput": 0.4030416917168955, + "input_token_throughput": 29712.253665454125, + "output_token_throughput": 51.32735944014665, + "total_token_throughput": 29763.581024894273, + "total_input_tokens": 1474401, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 22010.51428525243, + "p50": 23559.51010050194, + "p90": 31866.07759631006, + "p95": 32508.359657706755, + "p99": 36603.693813944985 + }, + "ttft_ms": { + "mean": 16855.99723370542, + "p50": 21033.618335000938, + "p90": 23834.377329907147, + "p95": 24693.19532076188, + "p99": 27039.11317213962 + }, + "tpot_ms": { + "mean": 59.877913532559745, + "p50": 49.298212603349945, + "p90": 60.082798051918886, + "p95": 78.21795468785261, + "p99": 316.9470677312616 + }, + "itl_ms": { + "mean": 219.8073648530414, + "p50": 271.98732501710765, + "p90": 333.0672606069129, + "p95": 357.2517809981946, + "p99": 390.4315355594735 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_10_131072_256.jsonl" + }, + { + "name": "c10_i131072_o4096", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 131072, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 19.08237444801489, + "request_throughput": 1.048087598033722, + "input_token_throughput": 77265.0701314259, + "output_token_throughput": 1998.231410031193, + "total_token_throughput": 79263.3015414571, + "total_input_tokens": 1474401, + "total_output_tokens": 38131, + "e2e_ms": { + "mean": 7316.236398999172, + "p50": 8444.335514504928, + "p90": 11945.644905421068, + "p95": 12401.944660821757, + "p99": 14731.589484146385 + }, + "ttft_ms": { + "mean": 727.3375002972898, + "p50": 600.3662900038762, + "p90": 1411.2119916011582, + "p95": 1543.0946984008187, + "p99": 1546.0051852866309 + }, + "tpot_ms": { + "mean": 3.335079117622894, + "p50": 3.428271593855011, + "p90": 4.07458192727982, + "p95": 4.806456682330838, + "p99": 4.841832708764732 + }, + "itl_ms": { + "mean": 20.002630943069565, + "p50": 15.639238990843296, + "p90": 22.070227304357104, + "p95": 34.30636346893152, + "p99": 165.97390236041974 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_10_131072_4096.jsonl" + }, + { + "name": "c2_i131072_o1024", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 1.7496567660127766, + "request_throughput": 2.2861626792753422, + "input_token_throughput": 138490.59124447187, + "output_token_throughput": 514.9581435067709, + "total_token_throughput": 139005.54938797862, + "total_input_tokens": 242311, + "total_output_tokens": 901, + "e2e_ms": { + "mean": 794.5647465166985, + "p50": 718.1648885161849, + "p90": 1252.8917442105014, + "p95": 1362.8063466094316, + "p99": 1450.738028528576 + }, + "ttft_ms": { + "mean": 447.72256826399826, + "p50": 459.6335765090771, + "p90": 665.1223058113828, + "p95": 679.8121634114067, + "p99": 691.5640494914259 + }, + "tpot_ms": { + "mean": 1.0965360940272797, + "p50": 1.3248563036316097, + "p90": 1.5558365349002174, + "p95": 1.5993635767368795, + "p99": 1.6341852102062093 + }, + "itl_ms": { + "mean": 11.097457431955263, + "p50": 8.672151016071439, + "p90": 10.805651010014122, + "p95": 16.695154790068035, + "p99": 34.131238552508904 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_2_131072_1024.jsonl" + }, + { + "name": "c2_i131072_o256", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 1.2918802190106362, + "request_throughput": 3.096262285882301, + "input_token_throughput": 187564.60268860657, + "output_token_throughput": 301.1115073020538, + "total_token_throughput": 187865.7141959086, + "total_input_tokens": 242311, + "total_output_tokens": 389, + "e2e_ms": { + "mean": 536.0358724938123, + "p50": 505.109255987918, + "p90": 705.8564373000991, + "p95": 747.5212021527112, + "p99": 780.8530140348012 + }, + "ttft_ms": { + "mean": 419.00899424945237, + "p50": 393.0140794982435, + "p90": 596.0212633013725, + "p95": 617.1667256538057, + "p99": 634.0830955357524 + }, + "tpot_ms": { + "mean": 0.9978323419928723, + "p50": 1.2476157901743834, + "p90": 1.389477322883724, + "p95": 1.4064074303054686, + "p99": 1.4199515162428642 + }, + "itl_ms": { + "mean": 9.17479329441181, + "p50": 9.940474992617965, + "p90": 10.147733002668247, + "p95": 16.18033648992423, + "p99": 18.293717512278818 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_2_131072_256.jsonl" + }, + { + "name": "c2_i131072_o4096", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 131072, + "output_len": 4096, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 8.443216187995858, + "request_throughput": 0.4737531185908753, + "input_token_throughput": 28698.897979718397, + "output_token_throughput": 1076.9592768367072, + "total_token_throughput": 29775.857256555104, + "total_input_tokens": 242311, + "total_output_tokens": 9093, + "e2e_ms": { + "mean": 4193.263632994785, + "p50": 4219.779900988215, + "p90": 6096.524830308044, + "p95": 6216.986983161768, + "p99": 6313.3567054447485 + }, + "ttft_ms": { + "mean": 325.5081640018034, + "p50": 354.36446650419384, + "p90": 538.5507072962355, + "p95": 540.4322966438485, + "p99": 541.937568121939 + }, + "tpot_ms": { + "mean": 1.7589143165131924, + "p50": 1.6630257624623965, + "p90": 1.963713117340703, + "p95": 2.027377708667831, + "p99": 2.078309381729534 + }, + "itl_ms": { + "mean": 10.065626301899645, + "p50": 9.872002003248781, + "p90": 10.049426596378908, + "p95": 10.112287598894909, + "p99": 16.348993644351122 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_2_131072_4096.jsonl" + }, + { + "name": "c5_i131072_o1024", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 3.6308107929944526, + "request_throughput": 2.754205760127936, + "input_token_throughput": 203409.3876290646, + "output_token_throughput": 1019.0561312473362, + "total_token_throughput": 204428.44376031193, + "total_input_tokens": 738541, + "total_output_tokens": 3700, + "e2e_ms": { + "mean": 1507.069445902016, + "p50": 1389.4180455099558, + "p90": 2304.1342217009515, + "p95": 2347.9891008522827, + "p99": 2383.0730041733477 + }, + "ttft_ms": { + "mean": 595.2868286985904, + "p50": 618.8132940005744, + "p90": 959.0717278100782, + "p95": 1080.0512569068812, + "p99": 1176.834880184324 + }, + "tpot_ms": { + "mean": 2.5274796112419384, + "p50": 2.131028194464456, + "p90": 3.7235818184212732, + "p95": 3.978484070795007, + "p99": 4.182405872693995 + }, + "itl_ms": { + "mean": 16.108580809175006, + "p50": 10.739890494733118, + "p90": 20.27784650272224, + "p95": 23.551662503450643, + "p99": 144.88391225168016 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_5_131072_1024.jsonl" + }, + { + "name": "c5_i131072_o256", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 22.894497758999933, + "request_throughput": 0.43678617042686396, + "input_token_throughput": 32258.449509322654, + "output_token_throughput": 60.975349391590214, + "total_token_throughput": 32319.424858714247, + "total_input_tokens": 738541, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 11223.107872297987, + "p50": 11322.323512998992, + "p90": 15126.79551889014, + "p95": 16310.015888452468, + "p99": 17256.592184102337 + }, + "ttft_ms": { + "mean": 5806.565075399703, + "p50": 7417.0587029948365, + "p90": 9662.113985186443, + "p95": 10128.318913598194, + "p99": 10501.282856327598 + }, + "tpot_ms": { + "mean": 37.59599971357063, + "p50": 44.28112107098226, + "p90": 59.72495268804145, + "p95": 60.864775370246505, + "p99": 61.77663351601055 + }, + "itl_ms": { + "mean": 216.66011608368717, + "p50": 273.5048829927109, + "p90": 346.02558271435555, + "p95": 364.7101309834396, + "p99": 392.47983420587843 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_5_131072_256.jsonl" + }, + { + "name": "c5_i131072_o4096", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 131072, + "output_len": 4096, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 14.276236372999847, + "request_throughput": 0.7004647260473114, + "input_token_throughput": 51732.19192397074, + "output_token_throughput": 1335.0857678461755, + "total_token_throughput": 53067.27769181692, + "total_input_tokens": 738541, + "total_output_tokens": 19060, + "e2e_ms": { + "mean": 6029.712195502361, + "p50": 7018.733311502729, + "p90": 9599.38420431863, + "p95": 10552.86903215601, + "p99": 11315.65689442592 + }, + "ttft_ms": { + "mean": 560.6182256044121, + "p50": 515.9745299897622, + "p90": 1041.3179694034625, + "p95": 1156.9083506998136, + "p99": 1249.3806557368953 + }, + "tpot_ms": { + "mean": 3.1961963242536493, + "p50": 3.206504942532288, + "p90": 4.1561452642836425, + "p95": 4.18660311174654, + "p99": 4.210969389716857 + }, + "itl_ms": { + "mean": 17.192877968244417, + "p50": 12.152664014138281, + "p90": 22.031987027730793, + "p95": 22.20277499873191, + "p99": 42.01793160755187 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_5_131072_4096.jsonl" + }, + { + "name": "c10_i262144_o1024", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 262144, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 97.72797075999551, + "request_throughput": 0.2046497010473782, + "input_token_throughput": 28498.70900153875, + "output_token_throughput": 96.78907611035751, + "total_token_throughput": 28595.498077649107, + "total_input_tokens": 2785121, + "total_output_tokens": 9459, + "e2e_ms": { + "mean": 45387.50252430036, + "p50": 44256.56188350695, + "p90": 58910.486319282805, + "p95": 70254.48793091347, + "p99": 91123.06626138739 + }, + "ttft_ms": { + "mean": 24591.82777870301, + "p50": 29435.68563000008, + "p90": 38974.941456803936, + "p95": 39321.29660845385, + "p99": 40783.01308169611 + }, + "tpot_ms": { + "mean": 48.18920425381463, + "p50": 55.989442050476555, + "p90": 66.05037972809353, + "p95": 67.60347733728547, + "p99": 68.1321781753959 + }, + "itl_ms": { + "mean": 244.08027261090186, + "p50": 291.6475804959191, + "p90": 424.5562060939847, + "p95": 458.57615228160245, + "p99": 522.1585231143399 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_10_262144_1024.jsonl" + }, + { + "name": "c10_i262144_o256", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 110.16605063999305, + "request_throughput": 0.18154413164321512, + "input_token_throughput": 25281.118673314148, + "output_token_throughput": 23.119645164763444, + "total_token_throughput": 25304.23831847891, + "total_input_tokens": 2785121, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 48223.85499715165, + "p50": 54898.71321149985, + "p90": 75289.47817128501, + "p95": 78728.48516409722, + "p99": 80196.261309619 + }, + "ttft_ms": { + "mean": 42248.62834745145, + "p50": 49703.44070899591, + "p90": 63688.684180699056, + "p95": 64654.52864100226, + "p99": 67877.62028660624 + }, + "tpot_ms": { + "mean": 52.054832429291196, + "p50": 49.41814168657996, + "p90": 67.86011650138555, + "p95": 73.84271063763447, + "p99": 151.27544932703617 + }, + "itl_ms": { + "mean": 255.34972855337435, + "p50": 286.28537949407473, + "p90": 402.0657468063291, + "p95": 437.00676189328067, + "p99": 492.95355229522085 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_10_262144_256.jsonl" + }, + { + "name": "c10_i262144_o4096", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 262144, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 108.96588197600795, + "request_throughput": 0.183543689431189, + "input_token_throughput": 25559.56919261413, + "output_token_throughput": 349.9352210850334, + "total_token_throughput": 25909.504413699164, + "total_input_tokens": 2785121, + "total_output_tokens": 38131, + "e2e_ms": { + "mean": 52210.569570692314, + "p50": 50657.05102798529, + "p90": 99869.24856970436, + "p95": 100105.12522634963, + "p99": 103440.52136287354 + }, + "ttft_ms": { + "mean": 13472.463327697187, + "p50": 12262.557213485707, + "p90": 25829.017210312315, + "p95": 33426.82968994487, + "p99": 37512.71495238295 + }, + "tpot_ms": { + "mean": 20.67259007483205, + "p50": 22.335029249613676, + "p90": 33.27995795684531, + "p95": 35.936134969934166, + "p99": 44.4381474839586 + }, + "itl_ms": { + "mean": 115.51536001699138, + "p50": 16.82196301408112, + "p90": 361.40161620569415, + "p95": 415.56643948715646, + "p99": 484.59458799276035 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_10_262144_4096.jsonl" + }, + { + "name": "c2_i262144_o1024", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 262144, + "output_len": 1024, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 16.191832354001235, + "request_throughput": 0.24703813086426518, + "input_token_throughput": 31154.905076283223, + "output_token_throughput": 55.64533897717573, + "total_token_throughput": 31210.550415260397, + "total_input_tokens": 504455, + "total_output_tokens": 901, + "e2e_ms": { + "mean": 7871.567239249998, + "p50": 7651.008348009782, + "p90": 13964.303008592105, + "p95": 14740.715870789425, + "p99": 15361.846160547282 + }, + "ttft_ms": { + "mean": 5853.5343170005945, + "p50": 5881.0410465084715, + "p90": 9820.337897987338, + "p95": 10466.903458483284, + "p99": 10984.155906880042 + }, + "tpot_ms": { + "mean": 14.357719716937018, + "p50": 3.7517795827225786, + "p90": 36.769081681037285, + "p95": 43.300111516563355, + "p99": 48.52493538498424 + }, + "itl_ms": { + "mean": 58.91917738659871, + "p50": 9.051853994606063, + "p90": 290.753513405798, + "p95": 320.6824104010593, + "p99": 350.4223257582632 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_2_262144_1024.jsonl" + }, + { + "name": "c2_i262144_o256", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 15.408737797988579, + "request_throughput": 0.25959296942038634, + "input_token_throughput": 32738.242847240246, + "output_token_throughput": 25.24541627613257, + "total_token_throughput": 32763.488263516378, + "total_input_tokens": 504455, + "total_output_tokens": 389, + "e2e_ms": { + "mean": 7672.816192993196, + "p50": 7643.010690488154, + "p90": 13423.424012202304, + "p95": 14088.425920606822, + "p99": 14620.42744733044 + }, + "ttft_ms": { + "mean": 5850.426714234345, + "p50": 5872.331345977727, + "p90": 9810.203461590574, + "p95": 10459.436945793275, + "p99": 10978.823733155441 + }, + "tpot_ms": { + "mean": 17.57577191929882, + "p50": 10.181436867008118, + "p90": 40.71390786208095, + "p95": 45.28395795219019, + "p99": 48.9399980242776 + }, + "itl_ms": { + "mean": 145.78776275971904, + "p50": 18.701617504120804, + "p90": 331.15072268410586, + "p95": 338.73794550599996, + "p99": 358.25617503229296 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_2_262144_256.jsonl" + }, + { + "name": "c2_i262144_o4096", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 262144, + "output_len": 4096, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 23.653920415003086, + "request_throughput": 0.16910516015192564, + "input_token_throughput": 21326.48589110991, + "output_token_throughput": 384.41830531536493, + "total_token_throughput": 21710.904196425276, + "total_input_tokens": 504455, + "total_output_tokens": 9093, + "e2e_ms": { + "mean": 11596.579911245499, + "p50": 11369.512781500816, + "p90": 14845.453088593786, + "p95": 14994.526062793739, + "p99": 15113.784442153701 + }, + "ttft_ms": { + "mean": 5487.6301339973, + "p50": 6178.046113491291, + "p90": 8360.751068586253, + "p95": 8709.001985285431, + "p99": 8987.602718644775 + }, + "tpot_ms": { + "mean": 3.2280306182866307, + "p50": 2.0858395742806968, + "p90": 5.745412071961834, + "p95": 6.43556928685951, + "p99": 6.987695058777652 + }, + "itl_ms": { + "mean": 16.012864087155666, + "p50": 10.026946503785439, + "p90": 10.195688009844162, + "p95": 10.277631241478957, + "p99": 302.1156209943001 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_2_262144_4096.jsonl" + }, + { + "name": "c5_i262144_o1024", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 262144, + "output_len": 1024, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 53.372761565988185, + "request_throughput": 0.18736148751899143, + "input_token_throughput": 26116.336481420964, + "output_token_throughput": 69.32375038202683, + "total_token_throughput": 26185.66023180299, + "total_input_tokens": 1393901, + "total_output_tokens": 3700, + "e2e_ms": { + "mean": 25295.536027196795, + "p50": 22571.09521899838, + "p90": 35288.65761580818, + "p95": 38376.68780889507, + "p99": 40847.11196336459 + }, + "ttft_ms": { + "mean": 13406.144849795965, + "p50": 14497.35130299814, + "p90": 18261.543534600056, + "p95": 22995.116275803583, + "p99": 26781.97446876642 + }, + "tpot_ms": { + "mean": 38.55896514833037, + "p50": 51.21804507602598, + "p90": 56.882710194174734, + "p95": 57.18789456377013, + "p99": 57.43204205944645 + }, + "itl_ms": { + "mean": 191.76376643215096, + "p50": 268.8099675142439, + "p90": 372.18185558740515, + "p95": 414.81327724904963, + "p99": 500.33757628465537 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_5_262144_1024.jsonl" + }, + { + "name": "c5_i262144_o256", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 52.2927886339894, + "request_throughput": 0.19123095671933957, + "input_token_throughput": 26655.702180204415, + "output_token_throughput": 26.695841558019804, + "total_token_throughput": 26682.398021762434, + "total_input_tokens": 1393901, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 23507.98017459747, + "p50": 25135.777328003314, + "p90": 32905.0044600066, + "p95": 35234.359566001505, + "p99": 37097.84365079744 + }, + "ttft_ms": { + "mean": 17996.009123898693, + "p50": 19306.366333999904, + "p90": 27986.28130730358, + "p95": 29425.380368152397, + "p99": 30576.659616831457 + }, + "tpot_ms": { + "mean": 41.735075029607174, + "p50": 50.43365461709989, + "p90": 58.2376680492912, + "p95": 60.423048465253345, + "p99": 62.17135279802306 + }, + "itl_ms": { + "mean": 244.9744939644976, + "p50": 286.5458000160288, + "p90": 376.96084639756003, + "p95": 426.482240395853, + "p99": 509.5562937820795 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_5_262144_256.jsonl" + }, + { + "name": "c5_i262144_o4096", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 262144, + "output_len": 4096, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 63.57839560499997, + "request_throughput": 0.15728613320361257, + "input_token_throughput": 21924.129835864875, + "output_token_throughput": 299.7873698860855, + "total_token_throughput": 22223.91720575096, + "total_input_tokens": 1393901, + "total_output_tokens": 19060, + "e2e_ms": { + "mean": 30807.30131380551, + "p50": 30501.745043002302, + "p90": 44364.00240242074, + "p95": 48506.30591822118, + "p99": 51820.14873086155 + }, + "ttft_ms": { + "mean": 10590.483214208507, + "p50": 10077.366048004478, + "p90": 16730.81445190764, + "p95": 22250.94545695318, + "p99": 26667.050260989636 + }, + "tpot_ms": { + "mean": 17.46113578915578, + "p50": 10.161597631207403, + "p90": 39.99704357805789, + "p95": 45.408594531434666, + "p99": 49.737835294136104 + }, + "itl_ms": { + "mean": 63.039536119419395, + "p50": 21.629737981129438, + "p90": 288.61644080025144, + "p95": 342.8714099165518, + "p99": 451.42972400062735 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_5_262144_4096.jsonl" + }, + { + "name": "c25_i65536_o1024", + "config": { + "phase": "dspark", + "concurrency": 25, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 50 + }, + "metrics": { + "success": 50, + "failed": 0, + "duration_s": 62.70547563501168, + "request_throughput": 0.79737853024246, + "input_token_throughput": 29366.669837870166, + "output_token_throughput": 412.73907482410215, + "total_token_throughput": 29779.408912694267, + "total_input_tokens": 1841451, + "total_output_tokens": 25881, + "e2e_ms": { + "mean": 29981.59460285853, + "p50": 29794.13296599523, + "p90": 48801.88996298821, + "p95": 57554.651349992484, + "p99": 60995.55153899796 + }, + "ttft_ms": { + "mean": 11692.068987198872, + "p50": 9282.525921502383, + "p90": 23589.7407762066, + "p95": 26078.115880547557, + "p99": 29135.598926548842 + }, + "tpot_ms": { + "mean": 42.82228902198422, + "p50": 45.10009716195819, + "p90": 53.573378286077535, + "p95": 56.18558621741822, + "p99": 133.88920998555187 + }, + "itl_ms": { + "mean": 197.6815361781092, + "p50": 264.1833170055179, + "p90": 296.874576000846, + "p95": 313.95786673238035, + "p99": 335.2813712626812 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_64k_0709_25_65536_1024.jsonl" + }, + { + "name": "c25_i65536_o256", + "config": { + "phase": "dspark", + "concurrency": 25, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 50 + }, + "metrics": { + "success": 50, + "failed": 0, + "duration_s": 59.856636811979115, + "request_throughput": 0.8353292577573201, + "input_token_throughput": 30764.3579405295, + "output_token_throughput": 98.78603802238068, + "total_token_throughput": 30863.143978551878, + "total_input_tokens": 1841451, + "total_output_tokens": 5913, + "e2e_ms": { + "mean": 24957.478126779897, + "p50": 26267.052594514098, + "p90": 34792.27077438264, + "p95": 37092.857118272506, + "p99": 39437.04205935879 + }, + "ttft_ms": { + "mean": 19438.036874281825, + "p50": 23692.236121496535, + "p90": 27105.122378107626, + "p95": 27635.110721291854, + "p99": 29212.481069455676 + }, + "tpot_ms": { + "mean": 51.68095443735579, + "p50": 49.096686958535194, + "p90": 58.59885411730775, + "p95": 65.5217898325625, + "p99": 134.9373555736381 + }, + "itl_ms": { + "mean": 251.79774711594382, + "p50": 267.6665144826984, + "p90": 299.91588299162686, + "p95": 320.7846462682937, + "p99": 347.9563715533004 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_64k_0709_25_65536_256.jsonl" + }, + { + "name": "c25_i65536_o4096", + "config": { + "phase": "dspark", + "concurrency": 25, + "input_len": 65536, + "output_len": 4096, + "dataset": "random", + "num_prompts": 50 + }, + "metrics": { + "success": 50, + "failed": 0, + "duration_s": 82.31095573300263, + "request_throughput": 0.6074525505716181, + "input_token_throughput": 22371.882134053136, + "output_token_throughput": 1409.204873969074, + "total_token_throughput": 23781.08700802221, + "total_input_tokens": 1841451, + "total_output_tokens": 115993, + "e2e_ms": { + "mean": 38164.69241847808, + "p50": 36029.353522491874, + "p90": 68485.24556439371, + "p95": 72245.56214315962, + "p99": 73496.425884809 + }, + "ttft_ms": { + "mean": 8084.046624338953, + "p50": 3264.411232987186, + "p90": 23582.51729948097, + "p95": 25048.60101691301, + "p99": 29143.3001965165 + }, + "tpot_ms": { + "mean": 14.71954587074255, + "p50": 13.270791017156972, + "p90": 22.879623646797487, + "p95": 30.574523231211494, + "p99": 42.06810737029239 + }, + "itl_ms": { + "mean": 72.87677341734114, + "p50": 23.865676994319074, + "p90": 272.30329640733544, + "p95": 286.6157213851693, + "p99": 309.7392923911684 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_64k_0709_25_65536_4096.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_long_context_matrix/run_bench.sh b/experiments/dsv4_h200_long_context_matrix/run_bench.sh index 5f9dc6f..fa8c89e 100755 --- a/experiments/dsv4_h200_long_context_matrix/run_bench.sh +++ b/experiments/dsv4_h200_long_context_matrix/run_bench.sh @@ -14,6 +14,13 @@ source "${SCRIPT_DIR}/config.env" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" +# Comma-separated list of backends to run. Default: sglang,vllm,dspark. +BACKENDS="${BACKENDS:-sglang,vllm,dspark}" + +backend_enabled() { + local backend="$1" + [[ ",${BACKENDS}," == *",${backend},"* ]] +} log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" mkdir -p "$log_dir_global" @@ -52,6 +59,8 @@ stop_server() { # Fallback cleanup. if [[ "$backend" == "sglang" ]]; then pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true + elif [[ "$backend" == "dspark" ]]; then + pkill -9 -f "vllm serve.*DeepSeek-V4-Flash-DSpark" 2>/dev/null || true else pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true fi @@ -62,13 +71,16 @@ start_server() { local backend="$1" local max_model_len="$2" local max_slots="$3" - local start_script + local start_script port if [[ "$backend" == "sglang" ]]; then start_script="${SGLANG_START_SCRIPT}" - local port="$SGLANG_PORT" + port="$SGLANG_PORT" + elif [[ "$backend" == "dspark" ]]; then + start_script="${DSPARK_START_SCRIPT}" + port="$DSPARK_PORT" else start_script="${VLLM_START_SCRIPT}" - local port="$VLLM_PORT" + port="$VLLM_PORT" fi log "starting ${backend} server (max_model_len=${max_model_len}, slots=${max_slots}) with ${start_script}" @@ -88,13 +100,15 @@ run_warmup() { local port if [[ "$backend" == "sglang" ]]; then port="$SGLANG_PORT" + elif [[ "$backend" == "dspark" ]]; then + port="$DSPARK_PORT" else port="$VLLM_PORT" fi log "warming up ${backend} (input=${input_len}, output=${output_len}, num=1)" "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ - --backend "$backend" \ + --backend vllm \ --port "$port" \ --input-len "$input_len" \ --output-len "$output_len" \ @@ -142,6 +156,8 @@ run_group() { local port if [[ "$backend" == "sglang" ]]; then port="$SGLANG_PORT" + elif [[ "$backend" == "dspark" ]]; then + port="$DSPARK_PORT" else port="$VLLM_PORT" fi @@ -161,9 +177,22 @@ run_group() { warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')" run_warmup "$backend" "$warmup_input_len" 256 + # Bench client backend: DSpark is served by vLLM API, so use vllm client. + local client_backend="$backend" + if [[ "$backend" == "dspark" ]]; then + client_backend="vllm" + fi + + # Output filename prefix: DSpark raw files must start with "vllm_" so that + # scripts/common/parse_backend.py (which only knows sglang/vllm) can parse them. + local file_prefix="${backend}" + if [[ "$backend" == "dspark" ]]; then + file_prefix="vllm_dspark" + fi + for scenario in "${scenarios_ref[@]}"; do read -r concurrency input_len output_len num_prompts <<< "$scenario" - output_file="${raw_dir}/${backend}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + output_file="${raw_dir}/${file_prefix}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${phase_log_dir}/${backend}_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log" if scenario_already_completed "$output_file" "$num_prompts"; then @@ -174,7 +203,7 @@ run_group() { log "running ${backend} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ - --backend "$backend" \ + --backend "$client_backend" \ --host 127.0.0.1 \ --port "$port" \ --dataset-name random \ @@ -200,8 +229,13 @@ run_group() { parse_backend() { local backend="$1" local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" - log "parsing ${backend} results in ${result_root}" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \ + # parse_backend.py only knows sglang/vllm; DSpark raw files use vllm client format. + local parser_backend="$backend" + if [[ "$backend" == "dspark" ]]; then + parser_backend="vllm" + fi + log "parsing ${backend} results in ${result_root} (parser_backend=${parser_backend})" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$parser_backend" \ >> "${result_root}/logs/parse.log" 2>&1 || { log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" } @@ -213,20 +247,28 @@ write_backend_metadata() { ensure_result_root "$result_root" local meta_json="${result_root}/results.json" - local env_path + local env_path model_path engine_name if [[ "$backend" == "sglang" ]]; then env_path="$VENV_SGLANG" + model_path="$MODEL_PATH" + engine_name="sglang" + elif [[ "$backend" == "dspark" ]]; then + env_path="$VENV_VLLM_DSPARK" + model_path="$DSPARK_MODEL_PATH" + engine_name="vllm-dspark" else env_path="$VENV_VLLM" + model_path="$MODEL_PATH" + engine_name="vllm" fi write_metadata_json \ "$meta_json" \ "${EXPERIMENT_NAME}_${backend}" \ "$RUN_ID" \ - "$MODEL_PATH" \ - "$backend" \ - "$backend" \ + "$model_path" \ + "${backend}" \ + "$engine_name" \ "$HARDWARE" \ "$ACCELERATOR" \ "$CHIP" \ @@ -254,11 +296,13 @@ write_backend_metadata() { # --------------------------------------------------------------------------- # Cleanup any leftovers. -stop_server sglang -stop_server vllm +for backend in sglang vllm dspark; do + backend_enabled "$backend" && stop_server "$backend" +done -write_backend_metadata sglang -write_backend_metadata vllm +for backend in sglang vllm dspark; do + backend_enabled "$backend" && write_backend_metadata "$backend" +done for group in "${CONTEXT_GROUPS[@]}"; do read -r group_label max_model_len max_slots <<< "$group" @@ -270,24 +314,28 @@ for group in "${CONTEXT_GROUPS[@]}"; do continue fi - # Run SGLang for this group. - run_group sglang "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name" - - # Run vLLM for this group. - run_group vllm "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name" + for backend in sglang vllm dspark; do + backend_enabled "$backend" && run_group "$backend" "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name" + done done # Parse and compare. -parse_backend sglang -parse_backend vllm +for backend in sglang vllm dspark; do + backend_enabled "$backend" && parse_backend "$backend" +done -log "generating comparison report" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \ - --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ - --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ - --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ - >> "${log_dir_global}/compare.log" 2>&1 || { - log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" - } +if backend_enabled sglang && backend_enabled vllm && backend_enabled dspark; then + log "generating three-way comparison report" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ + --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ + --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ + --dspark "${RESULT_BASE}/${RUN_ID}/dspark" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } +else + log "skipping three-way comparison (not all backends enabled: BACKENDS=${BACKENDS})" +fi log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_long_context_matrix/start_dspark.sh b/experiments/dsv4_h200_long_context_matrix/start_dspark.sh new file mode 100755 index 0000000..6c9ee56 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/start_dspark.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Start vLLM with DSpark speculative decoding for the long-context matrix. +# Usage: start_dspark.sh +set -e + +MAX_MODEL_LEN="${1:-80000}" +MAX_NUM_SEQS="${2:-64}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs tmp + +VENV="${VENV_VLLM_DSPARK}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/dsv4_h200_long_context_matrix_dspark_${MAX_MODEL_LEN}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_long_context_matrix_dspark.pid" + +rm -f "$PID_FILE" + +echo "=== Starting DeepSeek-V4-Flash-DSpark server (TP=$TP, max_model_len=$MAX_MODEL_LEN, max_num_seqs=$MAX_NUM_SEQS) ===" +echo "Model: $DSPARK_MODEL_PATH" +echo "Port: $DSPARK_PORT" +echo "Log: $LOG" + +nohup vllm serve "$DSPARK_MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --spec-method dspark \ + --spec-model "$DSPARK_MODEL_PATH" \ + --spec-tokens 5 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$DSPARK_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${DSPARK_PORT}/health" >/dev/null 2>&1; then + echo "DSpark server is ready at http://127.0.0.1:${DSPARK_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: DSpark server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: DSpark server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 From 8324d944f24f91c1b66bd455cba9b39b13414517 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 02:53:38 +0000 Subject: [PATCH 05/53] fix(p800): define RUN_ID/RESULT_ROOT/LOG_DIR in start_server.sh start_server.sh referenced LOG_DIR and RESULT_ROOT without defining them, causing an unbound-variable failure under set -u. Mirror the initialization from run_bench.sh so the script can run standalone. --- experiments/dsv4_p800_sglang/start_server.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experiments/dsv4_p800_sglang/start_server.sh b/experiments/dsv4_p800_sglang/start_server.sh index 63467b6..435865a 100755 --- a/experiments/dsv4_p800_sglang/start_server.sh +++ b/experiments/dsv4_p800_sglang/start_server.sh @@ -9,7 +9,11 @@ source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -log_init "${RESULT_ROOT:-/tmp/${EXPERIMENT_NAME}}/logs/start_server.log" +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +LOG_DIR="${RESULT_ROOT}/logs" + +log_init "${LOG_DIR}/start_server.log" log "starting server for ${EXPERIMENT_NAME}" log "model: ${MODEL_PATH}" log "port: ${PORT}" From 3694614dd96b310563c2b6d104f039b471e907b6 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 03:30:09 +0000 Subject: [PATCH 06/53] feat(p800): add max context length exploration experiment Add experiments/dsv4_p800_max_context_length/ to find the longest input context that P800 SGLang INT8 can serve for DeepSeek-V4-Flash-INT8. - Docker-based server start with dynamic --context-length. - Single-request, single-output-token probes for each candidate length. - Records success/failure, server args, and metrics per length. - Generates results.json and report.md summarizing the max supported length. - Smoke-tested at 4096 tokens successfully. --- .../dsv4_p800_max_context_length/README.md | 59 + .../dsv4_p800_max_context_length/config.env | 43 + .../extract_metrics.py | 75 + .../parse_results.py | 58 + .../results/20260709-032422/report.md | 18 + .../results/20260709-032422/results.json | 76 + .../dsv4_p800_max_context_length/run_bench.sh | 245 ++ .../start_server.sh | 133 + .../results/20260708-043404/results.json | 20 + .../results/20260708-043830/results.json | 20 + .../results/20260708-044535/results.json | 20 + .../results/20260708-045327/results.json | 20 + .../results/20260708-045554/results.json | 20 + .../results/20260708-045616/results.json | 20 + .../results/20260708-050025/results.json | 20 + .../results/20260708-050713/report.md | 16 + .../results/20260708-050713/results.json | 20 + .../results/20260708-050722/report.md | 16 + .../results/20260708-050722/results.json | 20 + .../results/20260708-050858/report.md | 16 + .../results/20260708-050858/results.json | 20 + .../results/20260708-051023/report.md | 16 + .../results/20260708-051023/results.json | 20 + .../results/20260708-051200/results.json | 20 + .../results/20260708-051316/report.md | 17 + .../results/20260708-051316/results.json | 2482 +++++++++++++++++ .../results/20260708-052659/report.md | 16 + .../results/20260708-052659/results.json | 20 + .../results/20260708-054332/results.json | 20 + 29 files changed, 3566 insertions(+) create mode 100644 experiments/dsv4_p800_max_context_length/README.md create mode 100644 experiments/dsv4_p800_max_context_length/config.env create mode 100755 experiments/dsv4_p800_max_context_length/extract_metrics.py create mode 100755 experiments/dsv4_p800_max_context_length/parse_results.py create mode 100644 experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md create mode 100644 experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json create mode 100755 experiments/dsv4_p800_max_context_length/run_bench.sh create mode 100755 experiments/dsv4_p800_max_context_length/start_server.sh create mode 100644 experiments/dsv4_p800_sglang/results/20260708-043404/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-043830/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-044535/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-045327/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-045554/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-045616/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050025/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050713/report.md create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050713/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050722/report.md create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050722/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050858/report.md create mode 100644 experiments/dsv4_p800_sglang/results/20260708-050858/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-051023/report.md create mode 100644 experiments/dsv4_p800_sglang/results/20260708-051023/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-051200/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-051316/report.md create mode 100644 experiments/dsv4_p800_sglang/results/20260708-051316/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-052659/report.md create mode 100644 experiments/dsv4_p800_sglang/results/20260708-052659/results.json create mode 100644 experiments/dsv4_p800_sglang/results/20260708-054332/results.json diff --git a/experiments/dsv4_p800_max_context_length/README.md b/experiments/dsv4_p800_max_context_length/README.md new file mode 100644 index 0000000..45c2252 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/README.md @@ -0,0 +1,59 @@ +# DSV4 P800 最大上下文长度探索实验 + +在 8x Kunlun P800 XPU 上，逐步增大输入长度，测试 SGLang serving `DeepSeek-V4-Flash-INT8` 时能稳定处理的最长上下文。 + +## 目的 + +- 找到 P800 + SGLang INT8 在 TP=8 配置下，单请求最多能支持多长的输入 token。 +- 每个长度只跑 1 个请求、输出 1 个 token，尽量减少 decode 阶段显存干扰。 +- 每次测试单独启动 server，并把 `context-length` 刚好设为 `target_len + CONTEXT_PAD`，避免预分配过大。 + +## 默认候选长度 + +```text +4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288 +``` + +可以通过环境变量覆盖： + +```bash +CANDIDATE_LENS=(4096 8192 16384) bash run_bench.sh +``` + +## 控制变量 + +| 维度 | 配置 | +|---|---| +| 硬件 | 8x Kunlun P800 XPU，TP=8 | +| 模型 | `DeepSeek-V4-Flash-INT8`（`w8a8_int8`） | +| 输入长度 | `--random-input-len $target_len --random-range-ratio 1.0` | +| 输出长度 | 1 | +| 并发 / 请求数 | 1 / 1 | +| 压测客户端 | 容器内 `sglang.bench_serving --backend sglang` | + +> `--random-range-ratio 1.0` 确保输入长度固定为 `target_len`，而不是默认的 `[1, target_len]` 均匀分布。 + +## 快速运行 + +```bash +bash experiments/dsv4_p800_max_context_length/run_bench.sh +``` + +结果保存在 `experiments/dsv4_p800_max_context_length/results//`： + +``` +results// +├── results.json +├── report.md +└── raw_outputs/ # gitignored +``` + +## 产物说明 + +- `results.json`：每个候选长度的尝试记录，包括 `success`、TTFT/TPOT/E2E 指标、`server_args`。 +- `report.md`：汇总表格，以及 SGLang INT8 最终支持的最大输入长度。 + +## 环境 + +- Docker image: `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202` +- Container Python: `/root/miniconda/envs/python310_torch25_cuda/bin/python` diff --git a/experiments/dsv4_p800_max_context_length/config.env b/experiments/dsv4_p800_max_context_length/config.env new file mode 100644 index 0000000..8366da6 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/config.env @@ -0,0 +1,43 @@ +# Common configuration for the max context length exploration on P800. +# All values can be overridden via environment variables. + +EXPERIMENT="dsv4_p800_max_context_length" +MODEL_NAME="DeepSeek-V4-Flash-INT8" +MODEL_PATH="/data1/models/DeepSeek-V4-Flash-INT8" +SERVED_MODEL_NAME="deepseek-v4-flash-int8" + +# Port must differ from other running experiments. +PORT="${PORT:-30013}" + +# We want exact input lengths, so fix the random range ratio to 1.0. +OUTPUT_LEN="${OUTPUT_LEN:-1}" +NUM_PROMPTS="${NUM_PROMPTS:-1}" +MAX_CONCURRENCY="${MAX_CONCURRENCY:-1}" +REQUEST_RATE="${REQUEST_RATE:-10000}" + +# Padding added to target length when setting server context length. +CONTEXT_PAD="${CONTEXT_PAD:-1024}" + +# Candidate input lengths (tokens). Override via CANDIDATE_LENS env var. +# Examples: +# CANDIDATE_LENS="4096 8192 16384" bash run_bench.sh +# CANDIDATE_LENS=(4096 8192 16384) bash run_bench.sh +if [[ -z "${CANDIDATE_LENS:-}" ]]; then + declare -a CANDIDATE_LENS=( + 4096 + 8192 + 16384 + 32768 + 65536 + 131072 + 262144 + 524288 + ) +elif ! declare -p CANDIDATE_LENS 2>/dev/null | grep -q '^declare -a'; then + # Normalize string-form arrays: remove surrounding parentheses and commas. + _CAND_STR="${CANDIDATE_LENS#(}" + _CAND_STR="${_CAND_STR%)}" + _CAND_STR="${_CAND_STR//,/}" + read -ra CANDIDATE_LENS <<< "$_CAND_STR" + unset _CAND_STR +fi diff --git a/experiments/dsv4_p800_max_context_length/extract_metrics.py b/experiments/dsv4_p800_max_context_length/extract_metrics.py new file mode 100755 index 0000000..c2579ce --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/extract_metrics.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Extract a compact metrics dict from a sglang.bench_serving JSONL output. + +Usage: + python3 extract_metrics.py +""" +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute(data: dict) -> dict: + return { + "success_count": data.get("completed", 0), + "total_count": len(data.get("input_lens", [])), + "duration_s": data.get("duration", 0.0), + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def main(): + path = Path(sys.argv[1]) + data = parse_jsonl(path) + if data is None: + raise SystemExit("No valid JSON found") + print(json.dumps(compute(data), indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_p800_max_context_length/parse_results.py b/experiments/dsv4_p800_max_context_length/parse_results.py new file mode 100755 index 0000000..715441f --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/parse_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Generate report.md from results.json for the max context length experiment. + +Usage: + python3 parse_results.py +""" +import json +import sys +from pathlib import Path + + +def main(): + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + results_json = result_root / "results.json" + report_path = result_root / "report.md" + + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + + meta = data["metadata"] + cfg = data["config"] + + with open(report_path, "w", encoding="utf-8") as f: + f.write("# P800 Max Context Length Exploration Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write(f"- Model: `{meta['model']}`\n") + f.write(f"- Hardware: {meta['hardware']}\n") + f.write(f"- TP: {cfg['tp']}, EP: {cfg['ep']}\n") + f.write(f"- Output len: {cfg['output_len']}, num prompts: {cfg['num_prompts']}, concurrency: {cfg['max_concurrency']}\n\n") + + f.write("## Per-attempt results\n\n") + f.write("| Target len | Max context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error |\n") + f.write("|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + max_supported = 0 + for s in data.get("scenarios", []): + target = s["target_len"] + status = "✅ pass" if s["success"] else "❌ fail" + metrics = s.get("metrics") or {} + ttft = metrics.get("ttft_ms", {}) + e2e = metrics.get("e2e_ms", {}) + f.write( + f"| {target} | {s['max_context_len']} | {status} | " + f"{ttft.get('mean', 0):.2f} | {ttft.get('p95', 0):.2f} | " + f"{e2e.get('mean', 0):.2f} | {e2e.get('p99', 0):.2f} | " + f"{s.get('error', '') or ''} |\n" + ) + if s["success"] and target > max_supported: + max_supported = target + + f.write("\n## Maximum supported input length\n\n") + f.write(f"- **sglang-xpu**: {max_supported} tokens\n\n") + + print(f"Wrote report to {report_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md b/experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md new file mode 100644 index 0000000..d2f0b79 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md @@ -0,0 +1,18 @@ +# P800 Max Context Length Exploration Report + +- Result root: `/data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-032422` +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Hardware: 8x Kunlun P800 XPU +- TP: 8, EP: 8 +- Output len: 1, num prompts: 1, concurrency: 1 + +## Per-attempt results + +| Target len | Max context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4096 | 5120 | ✅ pass | 231.33 | 0.00 | 231.34 | 231.34 | | + +## Maximum supported input length + +- **sglang-xpu**: 4096 tokens + diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json b/experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json new file mode 100644 index 0000000..60262e2 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json @@ -0,0 +1,76 @@ +{ + "metadata": { + "experiment": "dsv4_p800_max_context_length", + "run_id": "20260709-032422", + "timestamp": "2026-07-09T03:24:22+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_max_context_length/run_bench.sh", + "env": "", + "git_commit": "1d1c79d", + "git_dirty": "dirty", + "description": "P800 max context length exploration for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024 + }, + "scenarios": [ + { + "backend": "sglang", + "target_len": 4096, + "max_context_len": 5120, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 5120 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 5120", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.3442696970305406, + "request_throughput": 0.2990189460162034, + "input_token_throughput": 1224.781602882369, + "output_token_throughput": 0.2990189460162034, + "total_input_tokens": 4096, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 231.34461202425882, + "p50": 231.34461202425882, + "p90": 231.34461202425882, + "p95": 0.0, + "p99": 231.34461202425882 + }, + "ttft_ms": { + "mean": 231.33470199536532, + "p50": 231.33470199536532, + "p90": 0.0, + "p95": 0.0, + "p99": 231.33470199536532 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_max_context_length/run_bench.sh b/experiments/dsv4_p800_max_context_length/run_bench.sh new file mode 100755 index 0000000..2b5958d --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/run_bench.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# Max context length exploration for P800 SGLang INT8. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +LOG_DIR="${RESULT_ROOT}/logs" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}" +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "candidate_lengths=${CANDIDATE_LENS[*]}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + log "stopping container ${CONTAINER_NAME}" + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + pkill -9 -f 'sglang.launch_server' 2>/dev/null || true + sleep 2 +} + +start_server() { + local target_len="$1" + log "starting server (target_len=${target_len})" + bash "${SCRIPT_DIR}/start_server.sh" "$target_len" >> "${LOG_DIR}/start_server_${target_len}.log" 2>&1 +} + +build_server_args() { + local target_len="$1" + local max_context_len=$(( target_len + CONTEXT_PAD )) + echo "python -m sglang.launch_server --host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${max_context_len} --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${max_context_len}" +} + +append_scenario() { + local scenario_json="$1" + python3 - "$RESULT_JSON" "$scenario_json" <<'PY' +import json +import sys + +results_path, scenario_json = sys.argv[1], sys.argv[2] +with open(results_path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["scenarios"].append(json.loads(scenario_json)) +with open(results_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +# Build a scenario JSON blob safely with Python. +build_scenario_json() { + local target_len="$1" + local max_context_len="$2" + local success="$3" + local server_args="$4" + local error_msg="$5" + local metrics_json="${6:-null}" + + python3 - "$target_len" "$max_context_len" "$success" "$server_args" "$error_msg" "$metrics_json" <<'PY' +import json +import sys + +target_len, max_context_len, success, server_args, error_msg, metrics_json = sys.argv[1:7] +metrics = json.loads(metrics_json) if metrics_json not in ("null", "") else None +print(json.dumps({ + "backend": "sglang", + "target_len": int(target_len), + "max_context_len": int(max_context_len), + "success": success == "true", + "server_args": server_args, + "metrics": metrics, + "error": error_msg or None, +}, ensure_ascii=False)) +PY +} + +run_length() { + local target_len="$1" + local max_context_len=$(( target_len + CONTEXT_PAD )) + local output_file="${RAW_DIR}/sglang_ctx_${target_len}.jsonl" + local container_output="/tmp/bench_outputs/sglang_ctx_${target_len}.jsonl" + local detail_log="${LOG_DIR}/sglang_ctx_${target_len}.log" + + log "trying sglang target_len=${target_len} (context-length=${max_context_len})" + + stop_server + local server_args + server_args="$(build_server_args "$target_len")" + if ! start_server "$target_len"; then + local err="server failed to start" + log "ERROR: ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + return 1 + fi + + docker exec "$CONTAINER_NAME" mkdir -p "$(dirname "$container_output")" + + local bench_rc=0 + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "${CONTAINER_PYTHON}" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$PORT" \ + --model "$MODEL_PATH" \ + --dataset-name random \ + --dataset-path /workspace/dummy_sharegpt.json \ + --random-input-len "$target_len" \ + --random-output-len "$OUTPUT_LEN" \ + --random-range-ratio 1.0 \ + --num-prompts "$NUM_PROMPTS" \ + --max-concurrency "$MAX_CONCURRENCY" \ + --request-rate "$REQUEST_RATE" \ + --output-file "$container_output" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + if [[ "$bench_rc" -ne 0 ]]; then + local err="bench_serving exited with code ${bench_rc}; see ${detail_log}" + log "ERROR: sglang target_len=${target_len} failed; ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + stop_server + return 1 + fi + + docker cp "${CONTAINER_NAME}:${container_output}" "$output_file" || { + local err="failed to copy output from container" + log "ERROR: ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + stop_server + return 1 + } + + # Verify the request actually completed. + local completed + completed="$(python3 -c " +import json +with open('${output_file}') as f: + for line in f: + data = json.loads(line) + print(data.get('completed', 0)) + break +")" + if [[ "${completed:-0}" -lt "$NUM_PROMPTS" ]]; then + local err="only ${completed}/${NUM_PROMPTS} requests completed" + log "ERROR: sglang target_len=${target_len} ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + stop_server + return 1 + fi + + local metrics_json + metrics_json="$(python3 "${SCRIPT_DIR}/extract_metrics.py" "$output_file")" + + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "true" "$server_args" "" "$metrics_json")" + + log "sglang target_len=${target_len} succeeded" + stop_server +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +mkdir -p "$RAW_DIR" "$LOG_DIR" + +RESULT_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$RESULT_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang-xpu" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "P800 max context length exploration for DeepSeek-V4-Flash-INT8" + +# Embed config. +python3 - "$RESULT_JSON" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +stop_server + +for target_len in "${CANDIDATE_LENS[@]}"; do + if ! run_length "$target_len"; then + log "stopping exploration after first failure at target_len=${target_len}" + break + fi +done + +log "parsing results" +python3 "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parse_results.py failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_p800_max_context_length/start_server.sh b/experiments/dsv4_p800_max_context_length/start_server.sh new file mode 100755 index 0000000..0dbb973 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/start_server.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Start P800 SGLang INT8 server for a specific target context length. +set -Eeuo pipefail + +TARGET_LEN="${1:-}" +if [[ -z "$TARGET_LEN" ]]; then + echo "Usage: $0 " + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +MAX_CONTEXT_LEN=$(( TARGET_LEN + CONTEXT_PAD )) +CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}" +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" +PATCH_ROOT="${PATCH_ROOT:-${ROOT_DIR}/platforms/patches/kunlun_p800}" +RESULT_ROOT="${RESULT_ROOT:-/tmp/${EXPERIMENT}}" +SERVER_LOG="${RESULT_ROOT}/logs/server_${TARGET_LEN}.outer.log" + +mkdir -p "$(dirname "$SERVER_LOG")" + +log "starting P800 SGLang INT8 server (target_len=${TARGET_LEN}, context-length=${MAX_CONTEXT_LEN})" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "server log: ${SERVER_LOG}" + +# Stop any existing container with the same name. +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build device args. +device_args="" +for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" +done +device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + +# Environment variables required by the P800 SGLang INT8 image. +env_args=( + -e XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + -e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + -e CUDA_DEVICE_ORDER=OAM_ID + -e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 + -e XMLIR_FORCE_USE_XPU_GRAPH=1 + -e SGLANG_DSV4_MODE=2604 + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + -e SGLANG_NSA_DUAL_STREAM=true + -e SGLANG_NSA_QUANT_WQ_B_WK=false + -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + -e SGLANG_OPT_USE_TILELANG_MHC_PRE=1 + -e SGLANG_OPT_USE_TILELANG_MHC_POST=1 + -e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 + -e SGLANG_SET_CPU_AFFINITY=1 + -e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 + -e XSGL_INTERTYPE_BFP16=1 + -e ENABLE_FAST_BFP16_ATTN=1 + -e XSGL_USE_DEEP_GEMM_BMM=1 + -e XSGL_XDNN_QUANT=1 + -e XSGL_FUSE_RMS_NORM_QUANT=1 + -e XSGL_TRANSPOSE_MATMUL_WEIGHT=1 + -e XINFER_QUANT_SDNN=1 + -e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 + -e XSGL_EARLY_FIRST_TOKEN=1 + -e XSGL_ENABLE_TGEMM_FP16=1 + -e SGLANG_ENABLE_SPEC_V2=True + -e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + -e PYTHONDONTWRITEBYTECODE=1 + -e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops + -e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib + -e BKCL_TREE_THRESHOLD=1048576 + -e CUDA_ENABLE_P2P_NO_UVA=1 + -e NCCL_IB_GID_INDEX=3 + -e IS_DSV4=1 + -e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json + # INT8 specific + -e SGLANG_DSV4_FP4_EXPERTS=false + -e SGLANG_APPLY_CONFIG_BACKUP=auto + -e BKCL_ENABLE_XDR=1 + -e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 + -e BKCL_RDMA_VERBS=1 + -e XSGL_INT8_LM_HEAD=1 + -e SGLANG_P800_ALL_GATHER_FALLBACK=0 +) + +# Launch args. Based on the proven P800 INT8 command from server_docker.sh, +# but with dynamic --context-length and --max-prefill-tokens for the max-context +# test. Keep --max-running-requests at 64 because setting it to 1 causes +# req_to_token_pool allocation failures in this image. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" + +# Base64-encode the bootstrap command to avoid host-shell quoting issues. +server_cmd=$(cat </dev/null || true +/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q +/root/miniconda/envs/python310_torch25_cuda/bin/pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q +/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" ${launch_args} +EOF +) +server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0) + +patch_mounts=( + -v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro" + -v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro" +) + +docker run -d \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --network host \ + --ipc host \ + ${device_args} \ + -v "${MODEL_PATH}:/models:ro" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + "${patch_mounts[@]}" \ + "${env_args[@]}" \ + "${DOCKER_IMAGE}" \ + bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \ + >> "${SERVER_LOG}" 2>&1 + +log "container ${CONTAINER_NAME} started, waiting for health" +if health_check 127.0.0.1 "$PORT" 600; then + log "container ${CONTAINER_NAME} is healthy" +else + log "ERROR: container ${CONTAINER_NAME} failed health check" + exit 1 +fi diff --git a/experiments/dsv4_p800_sglang/results/20260708-043404/results.json b/experiments/dsv4_p800_sglang/results/20260708-043404/results.json new file mode 100644 index 0000000..79f632f --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-043404/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-043404", + "timestamp": "2026-07-08T04:34:04+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-043830/results.json b/experiments/dsv4_p800_sglang/results/20260708-043830/results.json new file mode 100644 index 0000000..5335eef --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-043830/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-043830", + "timestamp": "2026-07-08T04:38:30+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-044535/results.json b/experiments/dsv4_p800_sglang/results/20260708-044535/results.json new file mode 100644 index 0000000..94df31b --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-044535/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-044535", + "timestamp": "2026-07-08T04:45:35+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045327/results.json b/experiments/dsv4_p800_sglang/results/20260708-045327/results.json new file mode 100644 index 0000000..f2b1bbe --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-045327/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-045327", + "timestamp": "2026-07-08T04:53:27+00:00", + "model": "/data1/models/deepseek-ai/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045554/results.json b/experiments/dsv4_p800_sglang/results/20260708-045554/results.json new file mode 100644 index 0000000..a95c6d2 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-045554/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-045554", + "timestamp": "2026-07-08T04:55:54+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045616/results.json b/experiments/dsv4_p800_sglang/results/20260708-045616/results.json new file mode 100644 index 0000000..70013bf --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-045616/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-045616", + "timestamp": "2026-07-08T04:56:16+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-050025/results.json b/experiments/dsv4_p800_sglang/results/20260708-050025/results.json new file mode 100644 index 0000000..5ec902a --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050025/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050025", + "timestamp": "2026-07-08T05:00:25+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-050713/report.md b/experiments/dsv4_p800_sglang/results/20260708-050713/report.md new file mode 100644 index 0000000..fe11a79 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050713/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-050713 +- **Timestamp**: 2026-07-08T05:07:13+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050713/results.json b/experiments/dsv4_p800_sglang/results/20260708-050713/results.json new file mode 100644 index 0000000..06ec92b --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050713/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050713", + "timestamp": "2026-07-08T05:07:13+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-050722/report.md b/experiments/dsv4_p800_sglang/results/20260708-050722/report.md new file mode 100644 index 0000000..9d497fe --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050722/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-050722 +- **Timestamp**: 2026-07-08T05:07:22+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050722/results.json b/experiments/dsv4_p800_sglang/results/20260708-050722/results.json new file mode 100644 index 0000000..75d0b0f --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050722/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050722", + "timestamp": "2026-07-08T05:07:22+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-050858/report.md b/experiments/dsv4_p800_sglang/results/20260708-050858/report.md new file mode 100644 index 0000000..7bfb506 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050858/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-050858 +- **Timestamp**: 2026-07-08T05:08:58+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050858/results.json b/experiments/dsv4_p800_sglang/results/20260708-050858/results.json new file mode 100644 index 0000000..7978092 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050858/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050858", + "timestamp": "2026-07-08T05:08:58+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-051023/report.md b/experiments/dsv4_p800_sglang/results/20260708-051023/report.md new file mode 100644 index 0000000..4baebbb --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051023/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-051023 +- **Timestamp**: 2026-07-08T05:10:23+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-051023/results.json b/experiments/dsv4_p800_sglang/results/20260708-051023/results.json new file mode 100644 index 0000000..e0330b9 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051023/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-051023", + "timestamp": "2026-07-08T05:10:23+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-051200/results.json b/experiments/dsv4_p800_sglang/results/20260708-051200/results.json new file mode 100644 index 0000000..b806584 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051200/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-051200", + "timestamp": "2026-07-08T05:12:00+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-051316/report.md b/experiments/dsv4_p800_sglang/results/20260708-051316/report.md new file mode 100644 index 0000000..4a72f6c --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051316/report.md @@ -0,0 +1,17 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-051316 +- **Timestamp**: 2026-07-08T05:13:16+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c32_i512_o256 | 32 | 512/256 | 10 | 0 | 1.87 | 262.65 | 506.4399079710711 | 507.2712521097855 | 21.948128755653826 | 29.154062995221466 | 5314.590020017349 | diff --git a/experiments/dsv4_p800_sglang/results/20260708-051316/results.json b/experiments/dsv4_p800_sglang/results/20260708-051316/results.json new file mode 100644 index 0000000..60b7466 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051316/results.json @@ -0,0 +1,2482 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-051316", + "timestamp": "2026-07-08T05:13:16+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [ + { + "name": "c32_i512_o256", + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "success": 10, + "failed": 0, + "duration_s": 5.35, + "request_throughput": 1.87, + "input_token_throughput": 470.87, + "output_token_throughput": 262.65, + "total_token_throughput": 733.53, + "accept_length": null, + "latencies": { + "e2e_ms": { + "mean": 3372.781996097183, + "p50": 3498.675397480838, + "p90": 5125.400093087228, + "p95": null, + "p99": 5314.590020017349 + }, + "ttft_ms": { + "mean": 478.8548628857825, + "p50": 506.4399079710711, + "p90": null, + "p95": null, + "p99": 507.2712521097855 + }, + "tpot_ms": { + "mean": 22.334078483797782, + "p50": 21.948128755653826, + "p90": null, + "p95": null, + "p99": 29.154062995221466 + }, + "itl_ms": { + "mean": 20.75981864563044, + "p50": 18.560815330905218, + "p90": null, + "p95": 30.416983005125076, + "p99": 76.09861597185954 + } + }, + "raw_requests": [ + { + "tag": null, + "backend": "sglang", + "dataset_name": "random", + "request_rate": Infinity, + "max_concurrency": 32, + "sharegpt_output_len": null, + "random_input_len": 512, + "random_output_len": 256, + "random_range_ratio": 0.0, + "server_info": { + "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_mode": "auto", + "tokenizer_worker_num": 1, + "skip_tokenizer_init": false, + "load_format": "auto", + "model_loader_extra_config": "{}", + "trust_remote_code": true, + "context_length": null, + "is_embedding": false, + "enable_multimodal": null, + "revision": null, + "model_impl": "auto", + "host": "0.0.0.0", + "port": 30000, + "fastapi_root_path": "", + "grpc_mode": false, + "skip_server_warmup": false, + "warmups": null, + "nccl_port": null, + "checkpoint_engine_wait_weights_before_ready": false, + "ssl_keyfile": null, + "ssl_certfile": null, + "ssl_ca_certs": null, + "ssl_keyfile_password": null, + "enable_ssl_refresh": false, + "dtype": "auto", + "quantization": "w8a8_int8", + "quantization_param_path": null, + "kv_cache_dtype": "float16", + "enable_fp32_lm_head": false, + "modelopt_quant": null, + "modelopt_checkpoint_restore_path": null, + "modelopt_checkpoint_save_path": null, + "modelopt_export_path": null, + "quantize_and_serve": false, + "rl_quant_profile": null, + "mem_fraction_static": 0.8, + "max_running_requests": 64, + "max_queued_requests": null, + "max_total_tokens": null, + "chunked_prefill_size": 8192, + "enable_dynamic_chunking": false, + "max_prefill_tokens": 16384, + "prefill_max_requests": null, + "custom_max_context_len": null, + "schedule_policy": "fcfs", + "enable_priority_scheduling": false, + "disable_priority_preemption": false, + "default_priority_value": null, + "abort_on_priority_when_disabled": false, + "schedule_low_priority_values_first": false, + "priority_scheduling_preemption_threshold": 10, + "schedule_conservativeness": 1.0, + "page_size": 256, + "swa_full_tokens_ratio": 0.1, + "disable_hybrid_swa_memory": false, + "radix_eviction_policy": "lru", + "enable_prefill_delayer": false, + "prefill_delayer_max_delay_passes": 30, + "prefill_delayer_token_usage_low_watermark": null, + "prefill_delayer_forward_passes_buckets": null, + "prefill_delayer_wait_seconds_buckets": null, + "device": "cuda", + "tp_size": 8, + "pp_size": 1, + "pp_max_micro_batch_size": null, + "pp_async_batch_depth": 0, + "stream_interval": 1, + "stream_response_default_include_usage": false, + "incremental_streaming_output": false, + "enable_streaming_session": false, + "random_seed": 228424360, + "constrained_json_whitespace_pattern": null, + "constrained_json_disable_any_whitespace": true, + "watchdog_timeout": 3000000.0, + "soft_watchdog_timeout": null, + "dist_timeout": null, + "download_dir": null, + "model_checksum": null, + "base_gpu_id": 0, + "gpu_id_step": 1, + "sleep_on_idle": false, + "use_ray": false, + "custom_sigquit_handler": null, + "log_level": "info", + "log_level_http": null, + "log_requests": false, + "log_requests_level": 2, + "log_requests_format": "text", + "log_requests_target": null, + "uvicorn_access_log_exclude_prefixes": [], + "crash_dump_folder": null, + "show_time_cost": false, + "enable_metrics": true, + "enable_mfu_metrics": false, + "enable_metrics_for_all_schedulers": false, + "tokenizer_metrics_custom_labels_header": "x-custom-labels", + "tokenizer_metrics_allowed_custom_labels": null, + "extra_metric_labels": null, + "bucket_time_to_first_token": null, + "bucket_inter_token_latency": null, + "bucket_e2e_request_latency": null, + "collect_tokens_histogram": false, + "prompt_tokens_buckets": null, + "generation_tokens_buckets": null, + "gc_warning_threshold_secs": 0.0, + "decode_log_interval": 40, + "enable_request_time_stats_logging": true, + "kv_events_config": null, + "enable_trace": false, + "otlp_traces_endpoint": "localhost:4317", + "export_metrics_to_file": false, + "export_metrics_to_file_dir": null, + "api_key": null, + "admin_api_key": null, + "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", + "weight_version": "default", + "chat_template": null, + "hf_chat_template_name": null, + "completion_template": null, + "file_storage_path": "sglang_storage", + "enable_cache_report": false, + "reasoning_parser": "deepseek-v4", + "tool_call_parser": "deepseekv4", + "tool_server": null, + "sampling_defaults": "model", + "dp_size": 1, + "load_balance_method": "round_robin", + "attn_cp_size": 1, + "moe_dp_size": 1, + "dist_init_addr": null, + "nnodes": 1, + "node_rank": 0, + "json_model_override_args": "{}", + "preferred_sampling_params": null, + "enable_lora": null, + "enable_lora_overlap_loading": null, + "max_lora_rank": null, + "lora_target_modules": null, + "lora_paths": null, + "max_loaded_loras": null, + "max_loras_per_batch": 8, + "lora_eviction_policy": "lru", + "lora_backend": "csgmv", + "max_lora_chunk_size": 16, + "experts_shared_outer_loras": null, + "attention_backend": "compressed", + "decode_attention_backend": null, + "prefill_attention_backend": null, + "sampling_backend": "flashinfer", + "grammar_backend": "xgrammar", + "mm_attention_backend": null, + "fp8_gemm_runner_backend": "auto", + "fp4_gemm_runner_backend": "auto", + "nsa_prefill_backend": "klxdsa", + "nsa_decode_backend": "klxdsa", + "disable_flashinfer_autotune": false, + "mamba_backend": "triton", + "speculative_algorithm": "EAGLE", + "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "speculative_draft_model_revision": null, + "speculative_draft_load_format": null, + "speculative_num_steps": 3, + "speculative_eagle_topk": 1, + "speculative_num_draft_tokens": 4, + "speculative_accept_threshold_single": 1.0, + "speculative_accept_threshold_acc": 1.0, + "speculative_token_map": null, + "speculative_attention_mode": "prefill", + "speculative_draft_attention_backend": null, + "speculative_moe_runner_backend": "auto", + "speculative_moe_a2a_backend": null, + "speculative_draft_model_quantization": "w8a8_int8", + "speculative_ngram_min_bfs_breadth": 1, + "speculative_ngram_max_bfs_breadth": 10, + "speculative_ngram_match_type": "BFS", + "speculative_ngram_max_trie_depth": 18, + "speculative_ngram_capacity": 10000000, + "enable_multi_layer_eagle": false, + "ep_size": 8, + "moe_a2a_backend": "none", + "moe_runner_backend": "auto", + "flashinfer_mxfp4_moe_precision": "default", + "enable_hisparse": false, + "enable_flashinfer_allreduce_fusion": false, + "enforce_disable_flashinfer_allreduce_fusion": false, + "enable_aiter_allreduce_fusion": false, + "deepep_mode": "auto", + "ep_num_redundant_experts": 0, + "ep_dispatch_algorithm": null, + "init_expert_location": "trivial", + "enable_eplb": false, + "eplb_algorithm": "auto", + "eplb_rebalance_num_iterations": 1000, + "eplb_rebalance_layers_per_chunk": null, + "eplb_min_rebalancing_utilization_threshold": 1.0, + "expert_distribution_recorder_mode": null, + "expert_distribution_recorder_buffer_size": 1000, + "enable_expert_distribution_metrics": false, + "deepep_config": null, + "moe_dense_tp_size": null, + "elastic_ep_backend": null, + "enable_elastic_expert_backup": false, + "mooncake_ib_device": null, + "max_mamba_cache_size": null, + "mamba_ssm_dtype": null, + "mamba_full_memory_ratio": 0.9, + "mamba_scheduler_strategy": "no_buffer", + "mamba_track_interval": 256, + "linear_attn_backend": "triton", + "linear_attn_decode_backend": null, + "linear_attn_prefill_backend": null, + "enable_hierarchical_cache": false, + "hicache_ratio": 2.0, + "hicache_size": 0, + "hicache_write_policy": "write_through", + "hicache_io_backend": "kernel", + "hicache_mem_layout": "page_first", + "hicache_storage_backend": null, + "hicache_storage_prefetch_policy": "best_effort", + "hicache_storage_backend_extra_config": null, + "hisparse_config": null, + "enable_lmcache": false, + "kt_weight_path": null, + "kt_method": "AMXINT4", + "kt_cpuinfer": null, + "kt_threadpool_count": 2, + "kt_num_gpu_experts": null, + "kt_max_deferred_experts_per_token": null, + "dllm_algorithm": null, + "dllm_algorithm_config": null, + "enable_double_sparsity": false, + "ds_channel_config_path": null, + "ds_heavy_channel_num": 32, + "ds_heavy_token_num": 256, + "ds_heavy_channel_type": "qk", + "ds_sparse_decode_threshold": 4096, + "cpu_offload_gb": 0, + "offload_group_size": -1, + "offload_num_in_group": 1, + "offload_prefetch_step": 1, + "offload_mode": "cpu", + "multi_item_scoring_delimiter": null, + "disable_radix_cache": false, + "cuda_graph_max_bs": 32, + "cuda_graph_bs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32 + ], + "disable_cuda_graph": false, + "disable_cuda_graph_padding": false, + "enable_profile_cuda_graph": false, + "enable_cudagraph_gc": false, + "enable_layerwise_nvtx_marker": false, + "enable_nccl_nvls": false, + "enable_symm_mem": false, + "disable_flashinfer_cutlass_moe_fp4_allgather": false, + "enable_tokenizer_batch_encode": false, + "disable_tokenizer_batch_decode": false, + "disable_outlines_disk_cache": false, + "disable_custom_all_reduce": true, + "enable_mscclpp": false, + "enable_torch_symm_mem": false, + "pre_warm_nccl": false, + "disable_overlap_schedule": false, + "enable_mixed_chunk": false, + "enable_dp_attention": false, + "enable_dp_lm_head": false, + "enable_two_batch_overlap": false, + "enable_single_batch_overlap": false, + "tbo_token_distribution_threshold": 0.48, + "enable_torch_compile": false, + "disable_piecewise_cuda_graph": true, + "enforce_piecewise_cuda_graph": false, + "enable_torch_compile_debug_mode": false, + "torch_compile_max_bs": 32, + "piecewise_cuda_graph_max_tokens": 8192, + "piecewise_cuda_graph_tokens": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 48, + 64, + 80, + 96, + 112, + 128, + 144, + 160, + 176, + 192, + 208, + 224, + 240, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512, + 576, + 640, + 704, + 768, + 832, + 896, + 960, + 1024, + 1280, + 1536, + 1792, + 2048, + 2304, + 2560, + 2816, + 3072, + 3328, + 3584, + 3840, + 4096, + 4608, + 5120, + 5632, + 6144, + 6656, + 7168, + 7680, + 8192 + ], + "piecewise_cuda_graph_compiler": "eager", + "torchao_config": "", + "enable_nan_detection": false, + "enable_p2p_check": false, + "triton_attention_reduce_in_fp32": false, + "triton_attention_num_kv_splits": 8, + "triton_attention_split_tile_size": null, + "num_continuous_decode_steps": 1, + "delete_ckpt_after_loading": false, + "enable_memory_saver": false, + "enable_weights_cpu_backup": false, + "enable_draft_weights_cpu_backup": false, + "allow_auto_truncate": false, + "enable_custom_logit_processor": false, + "flashinfer_mla_disable_ragged": false, + "disable_shared_experts_fusion": true, + "disable_chunked_prefix_cache": false, + "disable_fast_image_processor": false, + "keep_mm_feature_on_device": false, + "enable_return_hidden_states": false, + "enable_return_routed_experts": false, + "enable_return_indexer_topk": false, + "scheduler_recv_interval": 1, + "numa_node": null, + "enable_deterministic_inference": false, + "rl_on_policy_target": null, + "enable_attn_tp_input_scattered": false, + "gc_threshold": null, + "enable_nsa_prefill_context_parallel": false, + "nsa_prefill_cp_mode": "in-seq-split", + "enable_fused_qk_norm_rope": false, + "enable_precise_embedding_interpolation": false, + "enable_fused_moe_sum_all_reduce": false, + "enable_prefill_context_parallel": false, + "prefill_cp_mode": "in-seq-split", + "enable_dynamic_batch_tokenizer": false, + "dynamic_batch_tokenizer_batch_size": 32, + "dynamic_batch_tokenizer_batch_timeout": 0.002, + "debug_tensor_dump_output_folder": null, + "debug_tensor_dump_layers": null, + "debug_tensor_dump_input_file": null, + "debug_tensor_dump_inject": false, + "disaggregation_mode": "null", + "disaggregation_transfer_backend": "mooncake", + "disaggregation_bootstrap_port": 8998, + "disaggregation_ib_device": null, + "disaggregation_decode_enable_offload_kvcache": false, + "num_reserved_decode_tokens": 512, + "disaggregation_decode_polling_interval": 1, + "encoder_only": false, + "language_only": false, + "encoder_transfer_backend": "zmq_to_scheduler", + "encoder_urls": [], + "enable_adaptive_dispatch_to_encoder": false, + "enable_over_encoding": false, + "enable_kv_mirror": false, + "custom_weight_loader": [], + "weight_loader_disable_mmap": false, + "remote_instance_weight_loader_seed_instance_ip": null, + "remote_instance_weight_loader_seed_instance_service_port": null, + "remote_instance_weight_loader_send_weights_group_ports": null, + "remote_instance_weight_loader_backend": "nccl", + "remote_instance_weight_loader_start_seed_via_transfer_engine": false, + "engine_info_bootstrap_port": 6789, + "modelexpress_config": null, + "enable_pdmux": false, + "pdmux_config_path": null, + "sm_group_num": 8, + "enable_broadcast_mm_inputs_process": false, + "enable_prefix_mm_cache": false, + "mm_enable_dp_encoder": false, + "mm_process_config": {}, + "limit_mm_data_per_request": null, + "enable_mm_global_cache": false, + "decrypted_config_file": null, + "decrypted_draft_config_file": null, + "forward_hooks": null, + "index_cache_topk_freq": 1, + "index_cache_pattern": null, + "status": "ready", + "max_total_num_tokens": 1593344, + "max_req_input_len": 1048570, + "internal_states": [ + { + "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_mode": "auto", + "tokenizer_worker_num": 1, + "skip_tokenizer_init": false, + "load_format": "auto", + "model_loader_extra_config": "{}", + "trust_remote_code": true, + "context_length": 1048576, + "is_embedding": false, + "enable_multimodal": null, + "revision": null, + "model_impl": "auto", + "host": "0.0.0.0", + "port": 30000, + "fastapi_root_path": "", + "grpc_mode": false, + "skip_server_warmup": false, + "warmups": null, + "nccl_port": null, + "checkpoint_engine_wait_weights_before_ready": false, + "ssl_keyfile": null, + "ssl_certfile": null, + "ssl_ca_certs": null, + "ssl_keyfile_password": null, + "enable_ssl_refresh": false, + "dtype": "auto", + "quantization": "w8a8_int8", + "quantization_param_path": null, + "kv_cache_dtype": "float16", + "enable_fp32_lm_head": false, + "modelopt_quant": null, + "modelopt_checkpoint_restore_path": null, + "modelopt_checkpoint_save_path": null, + "modelopt_export_path": null, + "quantize_and_serve": false, + "rl_quant_profile": null, + "mem_fraction_static": 0.8, + "max_running_requests": 64, + "max_queued_requests": null, + "max_total_tokens": null, + "chunked_prefill_size": 8192, + "enable_dynamic_chunking": false, + "max_prefill_tokens": 16384, + "prefill_max_requests": null, + "custom_max_context_len": null, + "schedule_policy": "fcfs", + "enable_priority_scheduling": false, + "disable_priority_preemption": false, + "default_priority_value": null, + "abort_on_priority_when_disabled": false, + "schedule_low_priority_values_first": false, + "priority_scheduling_preemption_threshold": 10, + "schedule_conservativeness": 1.0, + "page_size": 256, + "swa_full_tokens_ratio": 0.1, + "disable_hybrid_swa_memory": false, + "radix_eviction_policy": "lru", + "enable_prefill_delayer": false, + "prefill_delayer_max_delay_passes": 30, + "prefill_delayer_token_usage_low_watermark": null, + "prefill_delayer_forward_passes_buckets": null, + "prefill_delayer_wait_seconds_buckets": null, + "device": "cuda", + "tp_size": 8, + "pp_size": 1, + "pp_max_micro_batch_size": 64, + "pp_async_batch_depth": 0, + "stream_interval": 1, + "stream_response_default_include_usage": false, + "incremental_streaming_output": false, + "enable_streaming_session": false, + "random_seed": 228424360, + "constrained_json_whitespace_pattern": null, + "constrained_json_disable_any_whitespace": true, + "watchdog_timeout": 3000000.0, + "soft_watchdog_timeout": null, + "dist_timeout": null, + "download_dir": null, + "model_checksum": null, + "base_gpu_id": 0, + "gpu_id_step": 1, + "sleep_on_idle": false, + "use_ray": false, + "custom_sigquit_handler": null, + "log_level": "info", + "log_level_http": null, + "log_requests": false, + "log_requests_level": 2, + "log_requests_format": "text", + "log_requests_target": null, + "uvicorn_access_log_exclude_prefixes": [], + "crash_dump_folder": null, + "show_time_cost": false, + "enable_metrics": true, + "enable_mfu_metrics": false, + "enable_metrics_for_all_schedulers": false, + "tokenizer_metrics_custom_labels_header": "x-custom-labels", + "tokenizer_metrics_allowed_custom_labels": null, + "extra_metric_labels": null, + "bucket_time_to_first_token": null, + "bucket_inter_token_latency": null, + "bucket_e2e_request_latency": null, + "collect_tokens_histogram": false, + "prompt_tokens_buckets": null, + "generation_tokens_buckets": null, + "gc_warning_threshold_secs": 0.0, + "decode_log_interval": 40, + "enable_request_time_stats_logging": true, + "kv_events_config": null, + "enable_trace": false, + "otlp_traces_endpoint": "localhost:4317", + "export_metrics_to_file": false, + "export_metrics_to_file_dir": null, + "api_key": null, + "admin_api_key": null, + "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", + "weight_version": "default", + "chat_template": null, + "hf_chat_template_name": null, + "completion_template": null, + "file_storage_path": "sglang_storage", + "enable_cache_report": false, + "reasoning_parser": "deepseek-v4", + "tool_call_parser": "deepseekv4", + "tool_server": null, + "sampling_defaults": "model", + "dp_size": 1, + "load_balance_method": "round_robin", + "attn_cp_size": 1, + "moe_dp_size": 1, + "dist_init_addr": null, + "nnodes": 1, + "node_rank": 0, + "json_model_override_args": "{}", + "preferred_sampling_params": null, + "enable_lora": null, + "enable_lora_overlap_loading": null, + "max_lora_rank": null, + "lora_target_modules": null, + "lora_paths": null, + "max_loaded_loras": null, + "max_loras_per_batch": 8, + "lora_eviction_policy": "lru", + "lora_backend": "csgmv", + "max_lora_chunk_size": 16, + "experts_shared_outer_loras": null, + "attention_backend": "compressed", + "decode_attention_backend": "compressed", + "prefill_attention_backend": "compressed", + "sampling_backend": "flashinfer", + "grammar_backend": "xgrammar", + "mm_attention_backend": null, + "fp8_gemm_runner_backend": "auto", + "fp4_gemm_runner_backend": "auto", + "nsa_prefill_backend": "klxdsa", + "nsa_decode_backend": "klxdsa", + "disable_flashinfer_autotune": false, + "mamba_backend": "triton", + "speculative_algorithm": "EAGLE", + "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "speculative_draft_model_revision": null, + "speculative_draft_load_format": null, + "speculative_num_steps": 3, + "speculative_eagle_topk": 1, + "speculative_num_draft_tokens": 4, + "speculative_accept_threshold_single": 1.0, + "speculative_accept_threshold_acc": 1.0, + "speculative_token_map": null, + "speculative_attention_mode": "prefill", + "speculative_draft_attention_backend": null, + "speculative_moe_runner_backend": "auto", + "speculative_moe_a2a_backend": null, + "speculative_draft_model_quantization": "w8a8_int8", + "speculative_ngram_min_bfs_breadth": 1, + "speculative_ngram_max_bfs_breadth": 10, + "speculative_ngram_match_type": "BFS", + "speculative_ngram_max_trie_depth": 18, + "speculative_ngram_capacity": 10000000, + "enable_multi_layer_eagle": false, + "ep_size": 8, + "moe_a2a_backend": "none", + "moe_runner_backend": "auto", + "flashinfer_mxfp4_moe_precision": "default", + "enable_hisparse": false, + "enable_flashinfer_allreduce_fusion": false, + "enforce_disable_flashinfer_allreduce_fusion": false, + "enable_aiter_allreduce_fusion": false, + "deepep_mode": "auto", + "ep_num_redundant_experts": 0, + "ep_dispatch_algorithm": null, + "init_expert_location": "trivial", + "enable_eplb": false, + "eplb_algorithm": "auto", + "eplb_rebalance_num_iterations": 1000, + "eplb_rebalance_layers_per_chunk": null, + "eplb_min_rebalancing_utilization_threshold": 1.0, + "expert_distribution_recorder_mode": null, + "expert_distribution_recorder_buffer_size": 1000, + "enable_expert_distribution_metrics": false, + "deepep_config": null, + "moe_dense_tp_size": null, + "elastic_ep_backend": null, + "enable_elastic_expert_backup": false, + "mooncake_ib_device": null, + "max_mamba_cache_size": null, + "mamba_ssm_dtype": null, + "mamba_full_memory_ratio": 0.9, + "mamba_scheduler_strategy": "no_buffer", + "mamba_track_interval": 256, + "linear_attn_backend": "triton", + "linear_attn_decode_backend": null, + "linear_attn_prefill_backend": null, + "enable_hierarchical_cache": false, + "hicache_ratio": 2.0, + "hicache_size": 0, + "hicache_write_policy": "write_through", + "hicache_io_backend": "kernel", + "hicache_mem_layout": "page_first", + "hicache_storage_backend": null, + "hicache_storage_prefetch_policy": "best_effort", + "hicache_storage_backend_extra_config": null, + "hisparse_config": null, + "enable_lmcache": false, + "kt_weight_path": null, + "kt_method": "AMXINT4", + "kt_cpuinfer": null, + "kt_threadpool_count": 2, + "kt_num_gpu_experts": null, + "kt_max_deferred_experts_per_token": null, + "dllm_algorithm": null, + "dllm_algorithm_config": null, + "enable_double_sparsity": false, + "ds_channel_config_path": null, + "ds_heavy_channel_num": 32, + "ds_heavy_token_num": 256, + "ds_heavy_channel_type": "qk", + "ds_sparse_decode_threshold": 4096, + "cpu_offload_gb": 0, + "offload_group_size": -1, + "offload_num_in_group": 1, + "offload_prefetch_step": 1, + "offload_mode": "cpu", + "multi_item_scoring_delimiter": null, + "disable_radix_cache": false, + "cuda_graph_max_bs": 32, + "cuda_graph_bs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32 + ], + "disable_cuda_graph": false, + "disable_cuda_graph_padding": false, + "enable_profile_cuda_graph": false, + "enable_cudagraph_gc": false, + "enable_layerwise_nvtx_marker": false, + "enable_nccl_nvls": false, + "enable_symm_mem": false, + "disable_flashinfer_cutlass_moe_fp4_allgather": false, + "enable_tokenizer_batch_encode": false, + "disable_tokenizer_batch_decode": false, + "disable_outlines_disk_cache": false, + "disable_custom_all_reduce": true, + "enable_mscclpp": false, + "enable_torch_symm_mem": false, + "pre_warm_nccl": false, + "disable_overlap_schedule": false, + "enable_mixed_chunk": false, + "enable_dp_attention": false, + "enable_dp_lm_head": false, + "enable_two_batch_overlap": false, + "enable_single_batch_overlap": false, + "tbo_token_distribution_threshold": 0.48, + "enable_torch_compile": false, + "disable_piecewise_cuda_graph": true, + "enforce_piecewise_cuda_graph": false, + "enable_torch_compile_debug_mode": false, + "torch_compile_max_bs": 32, + "piecewise_cuda_graph_max_tokens": 8192, + "piecewise_cuda_graph_tokens": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 48, + 64, + 80, + 96, + 112, + 128, + 144, + 160, + 176, + 192, + 208, + 224, + 240, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512, + 576, + 640, + 704, + 768, + 832, + 896, + 960, + 1024, + 1280, + 1536, + 1792, + 2048, + 2304, + 2560, + 2816, + 3072, + 3328, + 3584, + 3840, + 4096, + 4608, + 5120, + 5632, + 6144, + 6656, + 7168, + 7680, + 8192 + ], + "piecewise_cuda_graph_compiler": "eager", + "torchao_config": "", + "enable_nan_detection": false, + "enable_p2p_check": false, + "triton_attention_reduce_in_fp32": false, + "triton_attention_num_kv_splits": 8, + "triton_attention_split_tile_size": null, + "num_continuous_decode_steps": 1, + "delete_ckpt_after_loading": false, + "enable_memory_saver": false, + "enable_weights_cpu_backup": false, + "enable_draft_weights_cpu_backup": false, + "allow_auto_truncate": false, + "enable_custom_logit_processor": false, + "flashinfer_mla_disable_ragged": false, + "disable_shared_experts_fusion": true, + "disable_chunked_prefix_cache": true, + "disable_fast_image_processor": false, + "keep_mm_feature_on_device": false, + "enable_return_hidden_states": false, + "enable_return_routed_experts": false, + "enable_return_indexer_topk": false, + "scheduler_recv_interval": 1, + "numa_node": null, + "enable_deterministic_inference": false, + "rl_on_policy_target": null, + "enable_attn_tp_input_scattered": false, + "gc_threshold": null, + "enable_nsa_prefill_context_parallel": false, + "nsa_prefill_cp_mode": "in-seq-split", + "enable_fused_qk_norm_rope": false, + "enable_precise_embedding_interpolation": false, + "enable_fused_moe_sum_all_reduce": false, + "enable_prefill_context_parallel": false, + "prefill_cp_mode": "in-seq-split", + "enable_dynamic_batch_tokenizer": false, + "dynamic_batch_tokenizer_batch_size": 32, + "dynamic_batch_tokenizer_batch_timeout": 0.002, + "debug_tensor_dump_output_folder": null, + "debug_tensor_dump_layers": null, + "debug_tensor_dump_input_file": null, + "debug_tensor_dump_inject": false, + "disaggregation_mode": "null", + "disaggregation_transfer_backend": "mooncake", + "disaggregation_bootstrap_port": 8998, + "disaggregation_ib_device": null, + "disaggregation_decode_enable_offload_kvcache": false, + "num_reserved_decode_tokens": 512, + "disaggregation_decode_polling_interval": 1, + "encoder_only": false, + "language_only": false, + "encoder_transfer_backend": "zmq_to_scheduler", + "encoder_urls": [], + "enable_adaptive_dispatch_to_encoder": false, + "enable_over_encoding": false, + "enable_kv_mirror": false, + "custom_weight_loader": [], + "weight_loader_disable_mmap": false, + "remote_instance_weight_loader_seed_instance_ip": null, + "remote_instance_weight_loader_seed_instance_service_port": null, + "remote_instance_weight_loader_send_weights_group_ports": null, + "remote_instance_weight_loader_backend": "nccl", + "remote_instance_weight_loader_start_seed_via_transfer_engine": false, + "engine_info_bootstrap_port": 6789, + "modelexpress_config": null, + "enable_pdmux": false, + "pdmux_config_path": null, + "sm_group_num": 8, + "enable_broadcast_mm_inputs_process": false, + "enable_prefix_mm_cache": false, + "mm_enable_dp_encoder": false, + "mm_process_config": {}, + "limit_mm_data_per_request": null, + "enable_mm_global_cache": false, + "decrypted_config_file": null, + "decrypted_draft_config_file": null, + "forward_hooks": null, + "index_cache_topk_freq": 1, + "index_cache_pattern": null, + "use_mla_backend": false, + "_mx_config_cache": {}, + "_draft_pool_config": { + "full_max_total_num_tokens": 1593344, + "swa_max_total_num_tokens": 159232 + }, + "last_gen_throughput": 304.3209280340091, + "memory_usage": { + "weight": 37.94, + "kvcache": 0, + "token_capacity": 1593344, + "graph": 1.2 + }, + "effective_max_running_requests_per_dp": 64, + "avg_spec_accept_length": 2.8933333333333335 + } + ], + "version": "0.5.10+ee6948cf8" + }, + "duration": 5.34541754698148, + "completed": 10, + "total_input_tokens": 2517, + "total_input_text_tokens": 2517, + "total_input_vision_tokens": 0, + "total_output_tokens": 1404, + "total_output_tokens_retokenized": 1404, + "request_throughput": 1.870761247762007, + "input_throughput": 470.8706060616972, + "output_throughput": 262.6548791857858, + "total_throughput": 733.525485247483, + "mean_e2e_latency_ms": 3372.781996097183, + "median_e2e_latency_ms": 3498.675397480838, + "std_e2e_latency_ms": 1426.9563106485741, + "p90_e2e_latency_ms": 5125.400093087228, + "p99_e2e_latency_ms": 5314.590020017349, + "mean_ttft_ms": 478.8548628857825, + "median_ttft_ms": 506.4399079710711, + "std_ttft_ms": 82.89120389713447, + "p99_ttft_ms": 507.2712521097855, + "mean_tpot_ms": 22.334078483797782, + "median_tpot_ms": 21.948128755653826, + "std_tpot_ms": 3.1100348959353536, + "p99_tpot_ms": 29.154062995221466, + "mean_itl_ms": 20.75981864563044, + "median_itl_ms": 18.560815330905218, + "std_itl_ms": 11.544034313142475, + "p95_itl_ms": 30.416983005125076, + "p99_itl_ms": 76.09861597185954, + "concurrency": 6.309669855447998, + "accept_length": 2.8933333333333335, + "max_output_tokens_per_s": 396.0, + "max_concurrent_requests": 10, + "input_lens": [ + 38, + 236, + 397, + 73, + 256, + 394, + 204, + 134, + 336, + 449 + ], + "output_lens": [ + 145, + 130, + 205, + 72, + 238, + 253, + 135, + 26, + 179, + 21 + ], + "ttfts": [ + 0.23018694896018133, + 0.5072830729768611, + 0.5071517300093547, + 0.506980306992773, + 0.5067555939895101, + 0.5065123789827339, + 0.5063674369594082, + 0.5061119620222598, + 0.5057157859555446, + 0.5054834120091982 + ], + "itls": [ + [ + 0.1265941219899105, + 0.1265941219899105, + 0.1265941219899105, + 0.1265941219899105, + 0.03003738750703633, + 0.03003738750703633, + 0.020075495665272076, + 0.020075495665272076, + 0.020075495665272076, + 0.020034517995857943, + 0.020034517995857943, + 0.020034517995857943, + 0.020025947674488027, + 0.020025947674488027, + 0.020025947674488027, + 0.01982437400147319, + 0.01982437400147319, + 0.01982437400147319, + 0.030580375489080325, + 0.030580375489080325, + 0.015474200758035295, + 0.015474200758035295, + 0.015474200758035295, + 0.015474200758035295, + 0.02975128698744811, + 0.02975128698744811, + 0.018891388666816056, + 0.018891388666816056, + 0.018891388666816056, + 0.030985667331454653, + 0.030985667331454653, + 0.030985667331454653, + 0.01450748200295493, + 0.01450748200295493, + 0.01450748200295493, + 0.01450748200295493, + 0.02843167950049974, + 0.02843167950049974, + 0.019097025661418836, + 0.019097025661418836, + 0.019097025661418836, + 0.019033391009240102, + 0.019033391009240102, + 0.019033391009240102, + 0.02840644898242317, + 0.02840644898242317, + 0.0287785355176311, + 0.0287785355176311, + 0.01427662099013105, + 0.01427662099013105, + 0.01427662099013105, + 0.01427662099013105, + 0.028972148516913876, + 0.028972148516913876, + 0.0198178769981799, + 0.0198178769981799, + 0.0198178769981799, + 0.019058960994395118, + 0.019058960994395118, + 0.019058960994395118, + 0.014342486509121954, + 0.014342486509121954, + 0.014342486509121954, + 0.014342486509121954, + 0.028646462480537593, + 0.028646462480537593, + 0.01456742275331635, + 0.01456742275331635, + 0.01456742275331635, + 0.01456742275331635, + 0.01943501700103904, + 0.01943501700103904, + 0.01943501700103904, + 0.02791283550322987, + 0.02791283550322987, + 0.013792212252155878, + 0.013792212252155878, + 0.013792212252155878, + 0.013792212252155878, + 0.018324121988068025, + 0.018324121988068025, + 0.018324121988068025, + 0.018392249670190115, + 0.018392249670190115, + 0.018392249670190115, + 0.018568886327557266, + 0.018568886327557266, + 0.018568886327557266, + 0.018351545673795044, + 0.018351545673795044, + 0.018351545673795044, + 0.02752352401148528, + 0.02752352401148528, + 0.013863828251487575, + 0.013863828251487575, + 0.013863828251487575, + 0.013863828251487575, + 0.018301511649042368, + 0.018301511649042368, + 0.018301511649042368, + 0.027680501516442746, + 0.027680501516442746, + 0.01841827033786103, + 0.01841827033786103, + 0.01841827033786103, + 0.01374593649234157, + 0.01374593649234157, + 0.01374593649234157, + 0.01374593649234157, + 0.018055193999316543, + 0.018055193999316543, + 0.018055193999316543, + 0.014001160496263765, + 0.014001160496263765, + 0.014001160496263765, + 0.014001160496263765, + 0.018516544679490227, + 0.018516544679490227, + 0.018516544679490227, + 0.018641953337161492, + 0.018641953337161492, + 0.018641953337161492, + 0.01858119231959184, + 0.01858119231959184, + 0.01858119231959184, + 0.018560895676879834, + 0.018560895676879834, + 0.018560895676879834, + 0.027825927507365122, + 0.027825927507365122, + 0.027690790477208793, + 0.027690790477208793, + 0.014033740997547284, + 0.014033740997547284, + 0.014033740997547284, + 0.014033740997547284, + 0.028556485020089895, + 0.028556485020089895, + 0.018292231000183772, + 0.018292231000183772, + 0.018292231000183772, + 0.03276129499621069, + 0.03276129499621069, + 0.03276129499621069 + ], + [ + 0.114176959003089, + 0.114176959003089, + 0.02001935833444198, + 0.02001935833444198, + 0.02001935833444198, + 0.020075112329019856, + 0.020075112329019856, + 0.020075112329019856, + 0.02003466134192422, + 0.02003466134192422, + 0.02003466134192422, + 0.020026450996131945, + 0.020026450996131945, + 0.020026450996131945, + 0.02973513101460412, + 0.02973513101460412, + 0.030412352993153036, + 0.030412352993153036, + 0.015535941754933447, + 0.015535941754933447, + 0.015535941754933447, + 0.015535941754933447, + 0.02982658799737692, + 0.02982658799737692, + 0.018890631986626733, + 0.018890631986626733, + 0.018890631986626733, + 0.03098562400555238, + 0.03098562400555238, + 0.03098562400555238, + 0.019343592672764014, + 0.019343592672764014, + 0.019343592672764014, + 0.018950106323851895, + 0.018950106323851895, + 0.018950106323851895, + 0.019099269004072994, + 0.019099269004072994, + 0.019099269004072994, + 0.019034251008027542, + 0.019034251008027542, + 0.019034251008027542, + 0.01420391199644655, + 0.01420391199644655, + 0.01420391199644655, + 0.01420391199644655, + 0.01439423524425365, + 0.01439423524425365, + 0.01439423524425365, + 0.01439423524425365, + 0.028566046996274963, + 0.028566046996274963, + 0.01930476234216864, + 0.01930476234216864, + 0.01930476234216864, + 0.019809113660206396, + 0.019809113660206396, + 0.019809113660206396, + 0.014296345761977136, + 0.014296345761977136, + 0.014296345761977136, + 0.014296345761977136, + 0.02867897297255695, + 0.02867897297255695, + 0.019099251677592594, + 0.019099251677592594, + 0.019099251677592594, + 0.019382666000941146, + 0.019382666000941146, + 0.019382666000941146, + 0.02924422750947997, + 0.02924422750947997, + 0.027913359983358532, + 0.027913359983358532, + 0.013784522248897702, + 0.013784522248897702, + 0.013784522248897702, + 0.013784522248897702, + 0.02746082251542248, + 0.02746082251542248, + 0.01840783333561073, + 0.01840783333561073, + 0.01840783333561073, + 0.01856821965581427, + 0.01856821965581427, + 0.01856821965581427, + 0.01835258233283336, + 0.01835258233283336, + 0.01835258233283336, + 0.018347706005442888, + 0.018347706005442888, + 0.018347706005442888, + 0.018484461004845798, + 0.018484461004845798, + 0.018484461004845798, + 0.018302411655895412, + 0.018302411655895412, + 0.018302411655895412, + 0.027679786493536085, + 0.027679786493536085, + 0.027627180505078286, + 0.027627180505078286, + 0.01375247175747063, + 0.01375247175747063, + 0.01375247175747063, + 0.01375247175747063, + 0.0270681754918769, + 0.0270681754918769, + 0.01866779732517898, + 0.01866779732517898, + 0.01866779732517898, + 0.018516544679490227, + 0.018516544679490227, + 0.018516544679490227, + 0.013980599993374199, + 0.013980599993374199, + 0.013980599993374199, + 0.013980599993374199, + 0.027871913509443402, + 0.027871913509443402, + 0.018560129004375387, + 0.018560129004375387, + 0.018560129004375387, + 0.018560815330905218, + 0.018560815330905218, + 0.018560815330905218, + 0.027690300485119224, + 0.027690300485119224, + 0.05595995101612061 + ], + [ + 0.07610034267418087, + 0.07610034267418087, + 0.07610034267418087, + 0.02001753800626223, + 0.02001753800626223, + 0.02001753800626223, + 0.020074322334645938, + 0.020074322334645938, + 0.020074322334645938, + 0.015025673492345959, + 0.015025673492345959, + 0.015025673492345959, + 0.015025673492345959, + 0.03003843649639748, + 0.03003843649639748, + 0.01982440067998444, + 0.01982440067998444, + 0.01982440067998444, + 0.02034523299274345, + 0.02034523299274345, + 0.02034523299274345, + 0.030977007001638412, + 0.030977007001638412, + 0.029805967991705984, + 0.029805967991705984, + 0.014167338755214587, + 0.014167338755214587, + 0.014167338755214587, + 0.014167338755214587, + 0.046478810982080176, + 0.046478810982080176, + 0.01934337633429095, + 0.01934337633429095, + 0.01934337633429095, + 0.0189525830016161, + 0.0189525830016161, + 0.0189525830016161, + 0.014323239010991529, + 0.014323239010991529, + 0.014323239010991529, + 0.014323239010991529, + 0.028550161485327408, + 0.028550161485327408, + 0.018938299675937742, + 0.018938299675937742, + 0.018938299675937742, + 0.019176949999139953, + 0.019176949999139953, + 0.019176949999139953, + 0.028566726978169754, + 0.028566726978169754, + 0.028956993512110785, + 0.028956993512110785, + 0.014857275251415558, + 0.014857275251415558, + 0.014857275251415558, + 0.014857275251415558, + 0.028592451504664496, + 0.028592451504664496, + 0.01912001532036811, + 0.01912001532036811, + 0.01912001532036811, + 0.019098964675019186, + 0.019098964675019186, + 0.019098964675019186, + 0.014537554743583314, + 0.014537554743583314, + 0.014537554743583314, + 0.014537554743583314, + 0.029245732497656718, + 0.029245732497656718, + 0.018609130017769832, + 0.018609130017769832, + 0.018609130017769832, + 0.018378569666917127, + 0.018378569666917127, + 0.018378569666917127, + 0.027461227495223284, + 0.027461227495223284, + 0.013806014991132542, + 0.013806014991132542, + 0.013806014991132542, + 0.013806014991132542, + 0.013926397252362221, + 0.013926397252362221, + 0.013926397252362221, + 0.013926397252362221, + 0.01835247567699601, + 0.01835247567699601, + 0.01835247567699601, + 0.01834764933058371, + 0.01834764933058371, + 0.01834764933058371, + 0.013863673244486563, + 0.013863673244486563, + 0.013863673244486563, + 0.013863673244486563, + 0.027453012502519414, + 0.027453012502519414, + 0.018453627669562895, + 0.018453627669562895, + 0.018453627669562895, + 0.018418410327285528, + 0.018418410327285528, + 0.018418410327285528, + 0.027505588514031842, + 0.027505588514031842, + 0.013533967750845477, + 0.013533967750845477, + 0.013533967750845477, + 0.013533967750845477, + 0.014000577997649089, + 0.014000577997649089, + 0.014000577997649089, + 0.014000577997649089, + 0.018516601334946852, + 0.018516601334946852, + 0.018516601334946852, + 0.018641606671735644, + 0.018641606671735644, + 0.018641606671735644, + 0.013935844239313155, + 0.013935844239313155, + 0.013935844239313155, + 0.013935844239313155, + 0.02784075849922374, + 0.02784075849922374, + 0.018561025344145794, + 0.018561025344145794, + 0.018561025344145794, + 0.01846017032706489, + 0.01846017032706489, + 0.01846017032706489, + 0.02801839599851519, + 0.02801839599851519, + 0.014315253254608251, + 0.014315253254608251, + 0.014315253254608251, + 0.014315253254608251, + 0.013719310751184821, + 0.013719310751184821, + 0.013719310751184821, + 0.013719310751184821, + 0.032736671001960836, + 0.032736671001960836, + 0.032736671001960836, + 0.01810533132326479, + 0.01810533132326479, + 0.01810533132326479, + 0.025817634508712217, + 0.025817634508712217, + 0.017274240672122687, + 0.017274240672122687, + 0.017274240672122687, + 0.012808780491468497, + 0.012808780491468497, + 0.012808780491468497, + 0.012808780491468497, + 0.017103748007987935, + 0.017103748007987935, + 0.017103748007987935, + 0.0260774539783597, + 0.0260774539783597, + 0.012770477012963966, + 0.012770477012963966, + 0.012770477012963966, + 0.012770477012963966, + 0.012633724749321118, + 0.012633724749321118, + 0.012633724749321118, + 0.012633724749321118, + 0.016969308664556593, + 0.016969308664556593, + 0.016969308664556593, + 0.017546184652019292, + 0.017546184652019292, + 0.017546184652019292, + 0.02627176101668738, + 0.02627176101668738, + 0.01677496867099156, + 0.01677496867099156, + 0.01677496867099156, + 0.012476857242290862, + 0.012476857242290862, + 0.012476857242290862, + 0.012476857242290862, + 0.016706993997407455, + 0.016706993997407455, + 0.016706993997407455, + 0.02511487199808471, + 0.02511487199808471, + 0.012620112000149675, + 0.012620112000149675, + 0.012620112000149675, + 0.012620112000149675, + 0.013204801754909568, + 0.013204801754909568, + 0.013204801754909568, + 0.013204801754909568, + 0.016931711327439796, + 0.016931711327439796, + 0.016931711327439796, + 0.025843739014817402, + 0.025843739014817402 + ], + [ + 0.07609820265012483, + 0.07609820265012483, + 0.07609820265012483, + 0.020024441337833803, + 0.020024441337833803, + 0.020024441337833803, + 0.015056459509651177, + 0.015056459509651177, + 0.015056459509651177, + 0.015056459509651177, + 0.03005173648125492, + 0.03005173648125492, + 0.015019418249721639, + 0.015019418249721639, + 0.015019418249721639, + 0.015019418249721639, + 0.019824344334968675, + 0.019824344334968675, + 0.019824344334968675, + 0.020387480326462537, + 0.020387480326462537, + 0.020387480326462537, + 0.020631657680496573, + 0.020631657680496573, + 0.020631657680496573, + 0.01487621849810239, + 0.01487621849810239, + 0.01487621849810239, + 0.01487621849810239, + 0.028335707494989038, + 0.028335707494989038, + 0.023239640504471026, + 0.023239640504471026, + 0.023239640504471026, + 0.023239640504471026, + 0.0193434563310196, + 0.0193434563310196, + 0.0193434563310196, + 0.028431184502551332, + 0.028431184502551332, + 0.014322734248708002, + 0.014322734248708002, + 0.014322734248708002, + 0.014322734248708002, + 0.01903337399320056, + 0.01903337399320056, + 0.01903337399320056, + 0.01893743300267185, + 0.01893743300267185, + 0.01893743300267185, + 0.01917593334413444, + 0.01917593334413444, + 0.01917593334413444, + 0.028568201989401132, + 0.028568201989401132, + 0.01930442900629714, + 0.01930442900629714, + 0.01930442900629714, + 0.014869755497784354, + 0.014869755497784354, + 0.014869755497784354, + 0.014869755497784354, + 0.028589651483343914, + 0.028589651483343914, + 0.019121012010145932, + 0.019121012010145932, + 0.019121012010145932, + 0.01909933132507528, + 0.01909933132507528, + 0.01909933132507528, + 0.0582002800074406 + ], + [ + 0.07609824265819043, + 0.07609824265819043, + 0.07609824265819043, + 0.015011793249868788, + 0.015011793249868788, + 0.015011793249868788, + 0.015011793249868788, + 0.020074732679252822, + 0.020074732679252822, + 0.020074732679252822, + 0.015025845990749076, + 0.015025845990749076, + 0.015025845990749076, + 0.015025845990749076, + 0.030038921511732042, + 0.030038921511732042, + 0.019824267326233286, + 0.019824267326233286, + 0.019824267326233286, + 0.020395867332505684, + 0.020395867332505684, + 0.020395867332505684, + 0.03094582151970826, + 0.03094582151970826, + 0.014877470748615451, + 0.014877470748615451, + 0.014877470748615451, + 0.014877470748615451, + 0.014167653993354179, + 0.014167653993354179, + 0.014167653993354179, + 0.014167653993354179, + 0.030986197331609826, + 0.030986197331609826, + 0.030986197331609826, + 0.01934319967404008, + 0.01934319967404008, + 0.01934319967404008, + 0.014215631992556155, + 0.014215631992556155, + 0.014215631992556155, + 0.014215631992556155, + 0.02864447850151919, + 0.02864447850151919, + 0.01903398767656957, + 0.01903398767656957, + 0.01903398767656957, + 0.018937196330322575, + 0.018937196330322575, + 0.018937196330322575, + 0.028763925016392022, + 0.028763925016392022, + 0.02856709697516635, + 0.02856709697516635, + 0.014478914003120735, + 0.014478914003120735, + 0.014478914003120735, + 0.014478914003120735, + 0.029714685515500605, + 0.029714685515500605, + 0.019061427660441648, + 0.019061427660441648, + 0.019061427660441648, + 0.019120605332621683, + 0.019120605332621683, + 0.019120605332621683, + 0.014324548756121658, + 0.014324548756121658, + 0.014324548756121658, + 0.014324548756121658, + 0.0290751994762104, + 0.0290751994762104, + 0.019497701665386558, + 0.019497701665386558, + 0.019497701665386558, + 0.01860897334214921, + 0.01860897334214921, + 0.01860897334214921, + 0.02756879449589178, + 0.02756879449589178, + 0.02748415298992768, + 0.02748415298992768, + 0.01379515475127846, + 0.01379515475127846, + 0.01379515475127846, + 0.01379515475127846, + 0.02785251452587545, + 0.02785251452587545, + 0.018352062324993312, + 0.018352062324993312, + 0.018352062324993312, + 0.01834822599388038, + 0.01834822599388038, + 0.01834822599388038, + 0.01386378325696569, + 0.01386378325696569, + 0.01386378325696569, + 0.01386378325696569, + 0.02745306250290014, + 0.02745306250290014, + 0.018453344334072124, + 0.018453344334072124, + 0.018453344334072124, + 0.018418603654329974, + 0.018418603654329974, + 0.018418603654329974, + 0.02750637350254692, + 0.02750637350254692, + 0.027067835500929505, + 0.027067835500929505, + 0.014000943003338762, + 0.014000943003338762, + 0.014000943003338762, + 0.014000943003338762, + 0.027774896996561438, + 0.027774896996561438, + 0.01864148332970217, + 0.01864148332970217, + 0.01864148332970217, + 0.018581362344169367, + 0.018581362344169367, + 0.018581362344169367, + 0.013920626748586074, + 0.013920626748586074, + 0.013920626748586074, + 0.013920626748586074, + 0.027841037983307615, + 0.027841037983307615, + 0.018460453681958217, + 0.018460453681958217, + 0.018460453681958217, + 0.018680113988618057, + 0.018680113988618057, + 0.018680113988618057, + 0.028647751518292353, + 0.028647751518292353, + 0.02743947648559697, + 0.02743947648559697, + 0.024545218504499644, + 0.024545218504499644, + 0.024545218504499644, + 0.024545218504499644, + 0.027157786476891488, + 0.027157786476891488, + 0.017211266347051907, + 0.017211266347051907, + 0.017211266347051907, + 0.017274483999547858, + 0.017274483999547858, + 0.017274483999547858, + 0.012808570492779836, + 0.012808570492779836, + 0.012808570492779836, + 0.012808570492779836, + 0.02565537701593712, + 0.02565537701593712, + 0.017431950002598267, + 0.017431950002598267, + 0.017431950002598267, + 0.016987925317759316, + 0.016987925317759316, + 0.016987925317759316, + 0.025259989517508075, + 0.025259989517508075, + 0.012736934004351497, + 0.012736934004351497, + 0.012736934004351497, + 0.012736934004351497, + 0.013151180988643318, + 0.013151180988643318, + 0.013151180988643318, + 0.013151180988643318, + 0.01752863434376195, + 0.01752863434376195, + 0.01752863434376195, + 0.01677370866915832, + 0.01677370866915832, + 0.01677370866915832, + 0.01247714224155061, + 0.01247714224155061, + 0.01247714224155061, + 0.01247714224155061, + 0.025057951017515734, + 0.025057951017515734, + 0.016742911325612415, + 0.016742911325612415, + 0.016742911325612415, + 0.016827172657940537, + 0.016827172657940537, + 0.016827172657940537, + 0.0176063990026402, + 0.0176063990026402, + 0.0176063990026402, + 0.01693138466604675, + 0.01693138466604675, + 0.01693138466604675, + 0.012955810001585633, + 0.012955810001585633, + 0.012955810001585633, + 0.012955810001585633, + 0.016913124340741586, + 0.016913124340741586, + 0.016913124340741586, + 0.015995322338615853, + 0.015995322338615853, + 0.015995322338615853, + 0.012007721743430011, + 0.012007721743430011, + 0.012007721743430011, + 0.012007721743430011, + 0.024051369488006458, + 0.024051369488006458, + 0.015974845349167783, + 0.015974845349167783, + 0.015974845349167783, + 0.015958434999144327, + 0.015958434999144327, + 0.015958434999144327, + 0.015792935659798484, + 0.015792935659798484, + 0.015792935659798484, + 0.01592183733979861, + 0.01592183733979861, + 0.01592183733979861, + 0.011986501252977178, + 0.011986501252977178, + 0.011986501252977178, + 0.011986501252977178, + 0.01585641598406558, + 0.01585641598406558, + 0.01585641598406558, + 0.015924710663966835, + 0.015924710663966835, + 0.015924710663966835, + 0.01634039101190865, + 0.01634039101190865, + 0.01634039101190865 + ], + [ + 0.057074277006904595, + 0.057074277006904595, + 0.057074277006904595, + 0.057074277006904595, + 0.030024631996639073, + 0.030024631996639073, + 0.020074642340963084, + 0.020074642340963084, + 0.020074642340963084, + 0.020034064655192196, + 0.020034064655192196, + 0.020034064655192196, + 0.015019408252555877, + 0.015019408252555877, + 0.015019408252555877, + 0.015019408252555877, + 0.02973640151321888, + 0.02973640151321888, + 0.020345932998073597, + 0.020345932998073597, + 0.020345932998073597, + 0.020652821326317888, + 0.020652821326317888, + 0.020652821326317888, + 0.014902361246640794, + 0.014902361246640794, + 0.014902361246640794, + 0.014902361246640794, + 0.014167511501000263, + 0.014167511501000263, + 0.014167511501000263, + 0.014167511501000263, + 0.04647939599817619, + 0.04647939599817619, + 0.019343009334988892, + 0.019343009334988892, + 0.019343009334988892, + 0.01895378966582939, + 0.01895378966582939, + 0.01895378966582939, + 0.014322384260594845, + 0.014322384260594845, + 0.014322384260594845, + 0.014322384260594845, + 0.02855006148456596, + 0.02855006148456596, + 0.018938185996375978, + 0.018938185996375978, + 0.018938185996375978, + 0.019176096674830962, + 0.019176096674830962, + 0.019176096674830962, + 0.019044518005102873, + 0.019044518005102873, + 0.019044518005102873, + 0.019305008987430483, + 0.019305008987430483, + 0.019305008987430483, + 0.014857510250294581, + 0.014857510250294581, + 0.014857510250294581, + 0.014857510250294581, + 0.0190613410086371, + 0.0190613410086371, + 0.0190613410086371, + 0.019120295338022213, + 0.019120295338022213, + 0.019120295338022213, + 0.0143242962367367, + 0.0143242962367367, + 0.0143242962367367, + 0.0143242962367367, + 0.029075569502310827, + 0.029075569502310827, + 0.019497531349770725, + 0.019497531349770725, + 0.019497531349770725, + 0.018608823651447892, + 0.018608823651447892, + 0.018608823651447892, + 0.018379226346344996, + 0.018379226346344996, + 0.018379226346344996, + 0.018320148325680446, + 0.018320148325680446, + 0.018320148325680446, + 0.01379665250715334, + 0.01379665250715334, + 0.01379665250715334, + 0.01379665250715334, + 0.01856846265339603, + 0.01856846265339603, + 0.01856846265339603, + 0.018352199345827103, + 0.018352199345827103, + 0.018352199345827103, + 0.018347878998611122, + 0.018347878998611122, + 0.018347878998611122, + 0.02772726200055331, + 0.02772726200055331, + 0.013726471239351667, + 0.013726471239351667, + 0.013726471239351667, + 0.013726471239351667, + 0.018453674002860982, + 0.018453674002860982, + 0.018453674002860982, + 0.027627600502455607, + 0.027627600502455607, + 0.013753056759014726, + 0.013753056759014726, + 0.013753056759014726, + 0.013753056759014726, + 0.013533932986319996, + 0.013533932986319996, + 0.013533932986319996, + 0.013533932986319996, + 0.018667823673846822, + 0.018667823673846822, + 0.018667823673846822, + 0.01851649465970695, + 0.01851649465970695, + 0.01851649465970695, + 0.027962180523900315, + 0.027962180523900315, + 0.01858143232918034, + 0.01858143232918034, + 0.01858143232918034, + 0.01392033399315551, + 0.01392033399315551, + 0.01392033399315551, + 0.01392033399315551, + 0.01856105534049372, + 0.01856105534049372, + 0.01856105534049372, + 0.027690295508364215, + 0.027690295508364215, + 0.014009808248374611, + 0.014009808248374611, + 0.014009808248374611, + 0.014009808248374611, + 0.014324050745926797, + 0.014324050745926797, + 0.014324050745926797, + 0.014324050745926797, + 0.018292734341230243, + 0.018292734341230243, + 0.018292734341230243, + 0.03272628099269544, + 0.03272628099269544, + 0.03272628099269544, + 0.027157796488609165, + 0.027157796488609165, + 0.017211413010954857, + 0.017211413010954857, + 0.017211413010954857, + 0.012955822996445931, + 0.012955822996445931, + 0.012955822996445931, + 0.012955822996445931, + 0.017078191003141303, + 0.017078191003141303, + 0.017078191003141303, + 0.02565545149263926, + 0.02565545149263926, + 0.026133759994991124, + 0.026133759994991124, + 0.012745351748890243, + 0.012745351748890243, + 0.012745351748890243, + 0.012745351748890243, + 0.02526378951733932, + 0.02526378951733932, + 0.016981848670790594, + 0.016981848670790594, + 0.016981848670790594, + 0.01753506433063497, + 0.01753506433063497, + 0.01753506433063497, + 0.013146101002348587, + 0.013146101002348587, + 0.013146101002348587, + 0.013146101002348587, + 0.025147862499579787, + 0.025147862499579787, + 0.016644779650960118, + 0.016644779650960118, + 0.016644779650960118, + 0.01670563401421532, + 0.01670563401421532, + 0.01670563401421532, + 0.025114176998613402, + 0.025114176998613402, + 0.025240779505111277, + 0.025240779505111277, + 0.01760628231568262, + 0.01760628231568262, + 0.01760628231568262, + 0.016931691012966137, + 0.016931691012966137, + 0.016931691012966137, + 0.01727279700571671, + 0.01727279700571671, + 0.01727279700571671, + 0.016912257318229724, + 0.016912257318229724, + 0.016912257318229724, + 0.01598840901472916, + 0.01598840901472916, + 0.01598840901472916, + 0.016010235665210832, + 0.016010235665210832, + 0.016010235665210832, + 0.016034339651620638, + 0.016034339651620638, + 0.016034339651620638, + 0.015974988675831508, + 0.015974988675831508, + 0.015974988675831508, + 0.023938142490806058, + 0.023938142490806058, + 0.023689378518611193, + 0.023689378518611193, + 0.01592199399601668, + 0.01592199399601668, + 0.01592199399601668, + 0.01598192499174426, + 0.01598192499174426, + 0.01598192499174426, + 0.01586189299511413, + 0.01586189299511413, + 0.01586189299511413, + 0.015926023673576612, + 0.015926023673576612, + 0.015926023673576612, + 0.016315380674010765, + 0.016315380674010765, + 0.016315380674010765, + 0.017069103002237778, + 0.017069103002237778, + 0.017069103002237778, + 0.015235118994799754, + 0.015235118994799754, + 0.015235118994799754, + 0.015159054659307003, + 0.015159054659307003, + 0.015159054659307003, + 0.022672360995784402, + 0.022672360995784402, + 0.023076677520293742, + 0.023076677520293742 + ], + [ + 0.07599885431894411, + 0.07599885431894411, + 0.07599885431894411, + 0.020025841678337503, + 0.020025841678337503, + 0.020025841678337503, + 0.020075395664510626, + 0.020075395664510626, + 0.020075395664510626, + 0.020034617996619392, + 0.020034617996619392, + 0.020034617996619392, + 0.020026374336642522, + 0.020026374336642522, + 0.020026374336642522, + 0.019823340660271544, + 0.019823340660271544, + 0.019823340660271544, + 0.030416983005125076, + 0.030416983005125076, + 0.03106233850121498, + 0.03106233850121498, + 0.019885292005104322, + 0.019885292005104322, + 0.019885292005104322, + 0.018892501985343795, + 0.018892501985343795, + 0.018892501985343795, + 0.030985270665648084, + 0.030985270665648084, + 0.030985270665648084, + 0.019343586017688114, + 0.019343586017688114, + 0.019343586017688114, + 0.018950182983341318, + 0.018950182983341318, + 0.018950182983341318, + 0.019099209011377145, + 0.019099209011377145, + 0.019099209011377145, + 0.01903421765503784, + 0.01903421765503784, + 0.01903421765503784, + 0.018937779338254284, + 0.018937779338254284, + 0.018937779338254284, + 0.02875587999005802, + 0.02875587999005802, + 0.02856260701082647, + 0.02856260701082647, + 0.019304305334420253, + 0.019304305334420253, + 0.019304305334420253, + 0.019817157338062923, + 0.019817157338062923, + 0.019817157338062923, + 0.019061451001713674, + 0.019061451001713674, + 0.019061451001713674, + 0.019113485332733642, + 0.019113485332733642, + 0.019113485332733642, + 0.019104608334600925, + 0.019104608334600925, + 0.019104608334600925, + 0.02907446399331093, + 0.02907446399331093, + 0.014618368746596389, + 0.014618368746596389, + 0.014618368746596389, + 0.014618368746596389, + 0.018610139998296898, + 0.018610139998296898, + 0.018610139998296898, + 0.02757459500571713, + 0.02757459500571713, + 0.027460306999273598, + 0.027460306999273598, + 0.01380542025435716, + 0.01380542025435716, + 0.01380542025435716, + 0.01380542025435716, + 0.027853768988279626, + 0.027853768988279626, + 0.0183525126776658, + 0.0183525126776658, + 0.0183525126776658, + 0.018347032329378028, + 0.018347032329378028, + 0.018347032329378028, + 0.018484470997160923, + 0.018484470997160923, + 0.018484470997160923, + 0.01830341199335332, + 0.01830341199335332, + 0.01830341199335332, + 0.01845283400810634, + 0.01845283400810634, + 0.01845283400810634, + 0.018417870334815234, + 0.018417870334815234, + 0.018417870334815234, + 0.027498273499077186, + 0.027498273499077186, + 0.027074395504314452, + 0.027074395504314452, + 0.018667700661656756, + 0.018667700661656756, + 0.018667700661656756, + 0.01851662133897965, + 0.01851662133897965, + 0.01851662133897965, + 0.018641199994211394, + 0.018641199994211394, + 0.018641199994211394, + 0.018580482341349125, + 0.018580482341349125, + 0.018580482341349125, + 0.018559868990754087, + 0.018559868990754087, + 0.018559868990754087, + 0.018557828676421195, + 0.018557828676421195, + 0.018557828676421195, + 0.018460286994619917, + 0.018460286994619917, + 0.018460286994619917, + 0.018637193328080077, + 0.018637193328080077, + 0.018637193328080077, + 0.028849135007476434, + 0.028849135007476434, + 0.027445936488220468, + 0.027445936488220468 + ], + [ + 0.07609848600501816, + 0.07609848600501816, + 0.07609848600501816, + 0.02001590799773112, + 0.02001590799773112, + 0.02001590799773112, + 0.020074725985371817, + 0.020074725985371817, + 0.020074725985371817, + 0.015025570755824447, + 0.015025570755824447, + 0.015025570755824447, + 0.015025570755824447, + 0.0200260576675646, + 0.0200260576675646, + 0.0200260576675646, + 0.014868118247250095, + 0.014868118247250095, + 0.014868118247250095, + 0.014868118247250095, + 0.01529539775219746, + 0.01529539775219746, + 0.01529539775219746, + 0.01529539775219746, + 0.0618297120090574 + ], + [ + 0.11415219900663942, + 0.11415219900663942, + 0.020037391669272136, + 0.020037391669272136, + 0.020037391669272136, + 0.015055509502417408, + 0.015055509502417408, + 0.015055509502417408, + 0.015055509502417408, + 0.020034414328013856, + 0.020034414328013856, + 0.020034414328013856, + 0.020025807665660977, + 0.020025807665660977, + 0.020025807665660977, + 0.01982415100792423, + 0.01982415100792423, + 0.01982415100792423, + 0.030516334489220753, + 0.030516334489220753, + 0.015466665499843657, + 0.015466665499843657, + 0.015466665499843657, + 0.015466665499843657, + 0.019884118674478184, + 0.019884118674478184, + 0.019884118674478184, + 0.028335032984614372, + 0.028335032984614372, + 0.023239655507495627, + 0.023239655507495627, + 0.023239655507495627, + 0.023239655507495627, + 0.014507339743431658, + 0.014507339743431658, + 0.014507339743431658, + 0.014507339743431658, + 0.01895076932851225, + 0.01895076932851225, + 0.01895076932851225, + 0.019099789011913042, + 0.019099789011913042, + 0.019099789011913042, + 0.028550306480610743, + 0.028550306480610743, + 0.018938216341969866, + 0.018938216341969866, + 0.018938216341969866, + 0.019193043660682935, + 0.019193043660682935, + 0.019193043660682935, + 0.019044294662307948, + 0.019044294662307948, + 0.019044294662307948, + 0.019304565348041553, + 0.019304565348041553, + 0.019304565348041553, + 0.014856965237413533, + 0.014856965237413533, + 0.014856965237413533, + 0.014856965237413533, + 0.014296558263595216, + 0.014296558263595216, + 0.014296558263595216, + 0.014296558263595216, + 0.01911949533193062, + 0.01911949533193062, + 0.01911949533193062, + 0.019099241665874917, + 0.019099241665874917, + 0.019099241665874917, + 0.029074459482217208, + 0.029074459482217208, + 0.01949675833263124, + 0.01949675833263124, + 0.01949675833263124, + 0.013956615002825856, + 0.013956615002825856, + 0.013956615002825856, + 0.013956615002825856, + 0.018378923006821424, + 0.018378923006821424, + 0.018378923006821424, + 0.027460777491796762, + 0.027460777491796762, + 0.013805887501803227, + 0.013805887501803227, + 0.013805887501803227, + 0.013805887501803227, + 0.013926447005360387, + 0.013926447005360387, + 0.013926447005360387, + 0.013926447005360387, + 0.01835258931775267, + 0.01835258931775267, + 0.01835258931775267, + 0.018347795664643247, + 0.018347795664643247, + 0.018347795664643247, + 0.013863318512449041, + 0.013863318512449041, + 0.013863318512449041, + 0.013863318512449041, + 0.027453502494608983, + 0.027453502494608983, + 0.01845339099721362, + 0.01845339099721362, + 0.01845339099721362, + 0.018418139991505694, + 0.018418139991505694, + 0.018418139991505694, + 0.027505743521032855, + 0.027505743521032855, + 0.027068145980592817, + 0.027068145980592817, + 0.0186672803441373, + 0.0186672803441373, + 0.0186672803441373, + 0.018516397996184725, + 0.018516397996184725, + 0.018516397996184725, + 0.018641243669359635, + 0.018641243669359635, + 0.018641243669359635, + 0.018581482329561066, + 0.018581482329561066, + 0.018581482329561066, + 0.01392028150439728, + 0.01392028150439728, + 0.01392028150439728, + 0.01392028150439728, + 0.027841487986734137, + 0.027841487986734137, + 0.018460020675168682, + 0.018460020675168682, + 0.018460020675168682, + 0.01867730700178072, + 0.01867730700178072, + 0.01867730700178072, + 0.028588780987774953, + 0.028588780987774953, + 0.018283454002812505, + 0.018283454002812505, + 0.018283454002812505, + 0.02455506350088399, + 0.02455506350088399, + 0.02455506350088399, + 0.02455506350088399, + 0.018118704669177532, + 0.018118704669177532, + 0.018118704669177532, + 0.012909814744489267, + 0.012909814744489267, + 0.012909814744489267, + 0.012909814744489267, + 0.012955870508449152, + 0.012955870508449152, + 0.012955870508449152, + 0.012955870508449152, + 0.025617786479415372, + 0.025617786479415372, + 0.01710334767509873, + 0.01710334767509873, + 0.01710334767509873, + 0.01738610933534801, + 0.01738610933534801, + 0.01738610933534801, + 0.025520398980006576, + 0.025520398980006576, + 0.016854186678150047, + 0.016854186678150047, + 0.016854186678150047, + 0.01274280650250148, + 0.01274280650250148, + 0.01274280650250148, + 0.01274280650250148, + 0.02618332498241216, + 0.02618332498241216 + ], + [ + 0.07610312267206609, + 0.07610312267206609, + 0.07610312267206609, + 0.01501510600792244, + 0.01501510600792244, + 0.01501510600792244, + 0.01501510600792244, + 0.020073268989411492, + 0.020073268989411492, + 0.020073268989411492, + 0.020034901332110167, + 0.020034901332110167, + 0.020034901332110167, + 0.01501977325824555, + 0.01501977325824555, + 0.01501977325824555, + 0.01501977325824555, + 0.02973549149464816, + 0.02973549149464816, + 0.06096841796534136 + ] + ], + "generated_texts": [ + " the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed", + " dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit volupt", + " quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog.", + "piciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The", + " The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspic", + "atem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", + " jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", + "nis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis", + " error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over", + " voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde om" + ], + "errors": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-052659/report.md b/experiments/dsv4_p800_sglang/results/20260708-052659/report.md new file mode 100644 index 0000000..a845a16 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-052659/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-052659 +- **Timestamp**: 2026-07-08T05:26:59+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 4ef6543 + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-052659/results.json b/experiments/dsv4_p800_sglang/results/20260708-052659/results.json new file mode 100644 index 0000000..ab11d28 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-052659/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-052659", + "timestamp": "2026-07-08T05:26:59+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "4ef6543", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-054332/results.json b/experiments/dsv4_p800_sglang/results/20260708-054332/results.json new file mode 100644 index 0000000..5c3f4eb --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-054332/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-054332", + "timestamp": "2026-07-08T05:43:32+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "4ef6543", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} From 06549d8faf07e32d02403c2d822bdffa738f8237 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 03:30:58 +0000 Subject: [PATCH 07/53] cleanup(p800): remove local debugging artifacts from dsv4_p800_sglang/results These were failed/intermediate smoke-test runs created while debugging the managed server startup. Keep the repository clean; new runs can be reproduced with experiments/dsv4_p800_sglang/run_bench.sh. --- .../results/20260708-043404/results.json | 20 - .../results/20260708-043830/results.json | 20 - .../results/20260708-044535/results.json | 20 - .../results/20260708-045327/results.json | 20 - .../results/20260708-045554/results.json | 20 - .../results/20260708-045616/results.json | 20 - .../results/20260708-050025/results.json | 20 - .../results/20260708-050713/report.md | 16 - .../results/20260708-050713/results.json | 20 - .../results/20260708-050722/report.md | 16 - .../results/20260708-050722/results.json | 20 - .../results/20260708-050858/report.md | 16 - .../results/20260708-050858/results.json | 20 - .../results/20260708-051023/report.md | 16 - .../results/20260708-051023/results.json | 20 - .../results/20260708-051200/results.json | 20 - .../results/20260708-051316/report.md | 17 - .../results/20260708-051316/results.json | 2482 ----------------- .../results/20260708-052659/report.md | 16 - .../results/20260708-052659/results.json | 20 - .../results/20260708-054332/results.json | 20 - 21 files changed, 2859 deletions(-) delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-043404/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-043830/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-044535/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-045327/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-045554/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-045616/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050025/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050713/report.md delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050713/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050722/report.md delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050722/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050858/report.md delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-050858/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-051023/report.md delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-051023/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-051200/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-051316/report.md delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-051316/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-052659/report.md delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-052659/results.json delete mode 100644 experiments/dsv4_p800_sglang/results/20260708-054332/results.json diff --git a/experiments/dsv4_p800_sglang/results/20260708-043404/results.json b/experiments/dsv4_p800_sglang/results/20260708-043404/results.json deleted file mode 100644 index 79f632f..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-043404/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-043404", - "timestamp": "2026-07-08T04:34:04+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-043830/results.json b/experiments/dsv4_p800_sglang/results/20260708-043830/results.json deleted file mode 100644 index 5335eef..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-043830/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-043830", - "timestamp": "2026-07-08T04:38:30+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-044535/results.json b/experiments/dsv4_p800_sglang/results/20260708-044535/results.json deleted file mode 100644 index 94df31b..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-044535/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-044535", - "timestamp": "2026-07-08T04:45:35+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045327/results.json b/experiments/dsv4_p800_sglang/results/20260708-045327/results.json deleted file mode 100644 index f2b1bbe..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-045327/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-045327", - "timestamp": "2026-07-08T04:53:27+00:00", - "model": "/data1/models/deepseek-ai/DeepSeek-V4-Flash", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045554/results.json b/experiments/dsv4_p800_sglang/results/20260708-045554/results.json deleted file mode 100644 index a95c6d2..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-045554/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-045554", - "timestamp": "2026-07-08T04:55:54+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045616/results.json b/experiments/dsv4_p800_sglang/results/20260708-045616/results.json deleted file mode 100644 index 70013bf..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-045616/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-045616", - "timestamp": "2026-07-08T04:56:16+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-050025/results.json b/experiments/dsv4_p800_sglang/results/20260708-050025/results.json deleted file mode 100644 index 5ec902a..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050025/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-050025", - "timestamp": "2026-07-08T05:00:25+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-050713/report.md b/experiments/dsv4_p800_sglang/results/20260708-050713/report.md deleted file mode 100644 index fe11a79..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050713/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# Benchmark Report: dsv4_p800_sglang - -## Metadata - -- **Run ID**: 20260708-050713 -- **Timestamp**: 2026-07-08T05:07:13+00:00 -- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 -- **Engine/Backend**: sglang-xpu / sglang -- **Hardware**: 8x Kunlun P800 XPU -- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 -- **Git Commit**: 2c332ff - -## Results - -| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050713/results.json b/experiments/dsv4_p800_sglang/results/20260708-050713/results.json deleted file mode 100644 index 06ec92b..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050713/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-050713", - "timestamp": "2026-07-08T05:07:13+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-050722/report.md b/experiments/dsv4_p800_sglang/results/20260708-050722/report.md deleted file mode 100644 index 9d497fe..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050722/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# Benchmark Report: dsv4_p800_sglang - -## Metadata - -- **Run ID**: 20260708-050722 -- **Timestamp**: 2026-07-08T05:07:22+00:00 -- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 -- **Engine/Backend**: sglang-xpu / sglang -- **Hardware**: 8x Kunlun P800 XPU -- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 -- **Git Commit**: 2c332ff - -## Results - -| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050722/results.json b/experiments/dsv4_p800_sglang/results/20260708-050722/results.json deleted file mode 100644 index 75d0b0f..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050722/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-050722", - "timestamp": "2026-07-08T05:07:22+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-050858/report.md b/experiments/dsv4_p800_sglang/results/20260708-050858/report.md deleted file mode 100644 index 7bfb506..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050858/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# Benchmark Report: dsv4_p800_sglang - -## Metadata - -- **Run ID**: 20260708-050858 -- **Timestamp**: 2026-07-08T05:08:58+00:00 -- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 -- **Engine/Backend**: sglang-xpu / sglang -- **Hardware**: 8x Kunlun P800 XPU -- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 -- **Git Commit**: 2c332ff - -## Results - -| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050858/results.json b/experiments/dsv4_p800_sglang/results/20260708-050858/results.json deleted file mode 100644 index 7978092..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-050858/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-050858", - "timestamp": "2026-07-08T05:08:58+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-051023/report.md b/experiments/dsv4_p800_sglang/results/20260708-051023/report.md deleted file mode 100644 index 4baebbb..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-051023/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# Benchmark Report: dsv4_p800_sglang - -## Metadata - -- **Run ID**: 20260708-051023 -- **Timestamp**: 2026-07-08T05:10:23+00:00 -- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 -- **Engine/Backend**: sglang-xpu / sglang -- **Hardware**: 8x Kunlun P800 XPU -- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 -- **Git Commit**: 2c332ff - -## Results - -| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-051023/results.json b/experiments/dsv4_p800_sglang/results/20260708-051023/results.json deleted file mode 100644 index e0330b9..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-051023/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-051023", - "timestamp": "2026-07-08T05:10:23+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-051200/results.json b/experiments/dsv4_p800_sglang/results/20260708-051200/results.json deleted file mode 100644 index b806584..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-051200/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-051200", - "timestamp": "2026-07-08T05:12:00+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} diff --git a/experiments/dsv4_p800_sglang/results/20260708-051316/report.md b/experiments/dsv4_p800_sglang/results/20260708-051316/report.md deleted file mode 100644 index 4a72f6c..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-051316/report.md +++ /dev/null @@ -1,17 +0,0 @@ -# Benchmark Report: dsv4_p800_sglang - -## Metadata - -- **Run ID**: 20260708-051316 -- **Timestamp**: 2026-07-08T05:13:16+00:00 -- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 -- **Engine/Backend**: sglang-xpu / sglang -- **Hardware**: 8x Kunlun P800 XPU -- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 -- **Git Commit**: 2c332ff - -## Results - -| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| c32_i512_o256 | 32 | 512/256 | 10 | 0 | 1.87 | 262.65 | 506.4399079710711 | 507.2712521097855 | 21.948128755653826 | 29.154062995221466 | 5314.590020017349 | diff --git a/experiments/dsv4_p800_sglang/results/20260708-051316/results.json b/experiments/dsv4_p800_sglang/results/20260708-051316/results.json deleted file mode 100644 index 60b7466..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-051316/results.json +++ /dev/null @@ -1,2482 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-051316", - "timestamp": "2026-07-08T05:13:16+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "2c332ff", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [ - { - "name": "c32_i512_o256", - "concurrency": 32, - "input_len": 512, - "output_len": 256, - "success": 10, - "failed": 0, - "duration_s": 5.35, - "request_throughput": 1.87, - "input_token_throughput": 470.87, - "output_token_throughput": 262.65, - "total_token_throughput": 733.53, - "accept_length": null, - "latencies": { - "e2e_ms": { - "mean": 3372.781996097183, - "p50": 3498.675397480838, - "p90": 5125.400093087228, - "p95": null, - "p99": 5314.590020017349 - }, - "ttft_ms": { - "mean": 478.8548628857825, - "p50": 506.4399079710711, - "p90": null, - "p95": null, - "p99": 507.2712521097855 - }, - "tpot_ms": { - "mean": 22.334078483797782, - "p50": 21.948128755653826, - "p90": null, - "p95": null, - "p99": 29.154062995221466 - }, - "itl_ms": { - "mean": 20.75981864563044, - "p50": 18.560815330905218, - "p90": null, - "p95": 30.416983005125076, - "p99": 76.09861597185954 - } - }, - "raw_requests": [ - { - "tag": null, - "backend": "sglang", - "dataset_name": "random", - "request_rate": Infinity, - "max_concurrency": 32, - "sharegpt_output_len": null, - "random_input_len": 512, - "random_output_len": 256, - "random_range_ratio": 0.0, - "server_info": { - "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", - "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", - "tokenizer_mode": "auto", - "tokenizer_worker_num": 1, - "skip_tokenizer_init": false, - "load_format": "auto", - "model_loader_extra_config": "{}", - "trust_remote_code": true, - "context_length": null, - "is_embedding": false, - "enable_multimodal": null, - "revision": null, - "model_impl": "auto", - "host": "0.0.0.0", - "port": 30000, - "fastapi_root_path": "", - "grpc_mode": false, - "skip_server_warmup": false, - "warmups": null, - "nccl_port": null, - "checkpoint_engine_wait_weights_before_ready": false, - "ssl_keyfile": null, - "ssl_certfile": null, - "ssl_ca_certs": null, - "ssl_keyfile_password": null, - "enable_ssl_refresh": false, - "dtype": "auto", - "quantization": "w8a8_int8", - "quantization_param_path": null, - "kv_cache_dtype": "float16", - "enable_fp32_lm_head": false, - "modelopt_quant": null, - "modelopt_checkpoint_restore_path": null, - "modelopt_checkpoint_save_path": null, - "modelopt_export_path": null, - "quantize_and_serve": false, - "rl_quant_profile": null, - "mem_fraction_static": 0.8, - "max_running_requests": 64, - "max_queued_requests": null, - "max_total_tokens": null, - "chunked_prefill_size": 8192, - "enable_dynamic_chunking": false, - "max_prefill_tokens": 16384, - "prefill_max_requests": null, - "custom_max_context_len": null, - "schedule_policy": "fcfs", - "enable_priority_scheduling": false, - "disable_priority_preemption": false, - "default_priority_value": null, - "abort_on_priority_when_disabled": false, - "schedule_low_priority_values_first": false, - "priority_scheduling_preemption_threshold": 10, - "schedule_conservativeness": 1.0, - "page_size": 256, - "swa_full_tokens_ratio": 0.1, - "disable_hybrid_swa_memory": false, - "radix_eviction_policy": "lru", - "enable_prefill_delayer": false, - "prefill_delayer_max_delay_passes": 30, - "prefill_delayer_token_usage_low_watermark": null, - "prefill_delayer_forward_passes_buckets": null, - "prefill_delayer_wait_seconds_buckets": null, - "device": "cuda", - "tp_size": 8, - "pp_size": 1, - "pp_max_micro_batch_size": null, - "pp_async_batch_depth": 0, - "stream_interval": 1, - "stream_response_default_include_usage": false, - "incremental_streaming_output": false, - "enable_streaming_session": false, - "random_seed": 228424360, - "constrained_json_whitespace_pattern": null, - "constrained_json_disable_any_whitespace": true, - "watchdog_timeout": 3000000.0, - "soft_watchdog_timeout": null, - "dist_timeout": null, - "download_dir": null, - "model_checksum": null, - "base_gpu_id": 0, - "gpu_id_step": 1, - "sleep_on_idle": false, - "use_ray": false, - "custom_sigquit_handler": null, - "log_level": "info", - "log_level_http": null, - "log_requests": false, - "log_requests_level": 2, - "log_requests_format": "text", - "log_requests_target": null, - "uvicorn_access_log_exclude_prefixes": [], - "crash_dump_folder": null, - "show_time_cost": false, - "enable_metrics": true, - "enable_mfu_metrics": false, - "enable_metrics_for_all_schedulers": false, - "tokenizer_metrics_custom_labels_header": "x-custom-labels", - "tokenizer_metrics_allowed_custom_labels": null, - "extra_metric_labels": null, - "bucket_time_to_first_token": null, - "bucket_inter_token_latency": null, - "bucket_e2e_request_latency": null, - "collect_tokens_histogram": false, - "prompt_tokens_buckets": null, - "generation_tokens_buckets": null, - "gc_warning_threshold_secs": 0.0, - "decode_log_interval": 40, - "enable_request_time_stats_logging": true, - "kv_events_config": null, - "enable_trace": false, - "otlp_traces_endpoint": "localhost:4317", - "export_metrics_to_file": false, - "export_metrics_to_file_dir": null, - "api_key": null, - "admin_api_key": null, - "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", - "weight_version": "default", - "chat_template": null, - "hf_chat_template_name": null, - "completion_template": null, - "file_storage_path": "sglang_storage", - "enable_cache_report": false, - "reasoning_parser": "deepseek-v4", - "tool_call_parser": "deepseekv4", - "tool_server": null, - "sampling_defaults": "model", - "dp_size": 1, - "load_balance_method": "round_robin", - "attn_cp_size": 1, - "moe_dp_size": 1, - "dist_init_addr": null, - "nnodes": 1, - "node_rank": 0, - "json_model_override_args": "{}", - "preferred_sampling_params": null, - "enable_lora": null, - "enable_lora_overlap_loading": null, - "max_lora_rank": null, - "lora_target_modules": null, - "lora_paths": null, - "max_loaded_loras": null, - "max_loras_per_batch": 8, - "lora_eviction_policy": "lru", - "lora_backend": "csgmv", - "max_lora_chunk_size": 16, - "experts_shared_outer_loras": null, - "attention_backend": "compressed", - "decode_attention_backend": null, - "prefill_attention_backend": null, - "sampling_backend": "flashinfer", - "grammar_backend": "xgrammar", - "mm_attention_backend": null, - "fp8_gemm_runner_backend": "auto", - "fp4_gemm_runner_backend": "auto", - "nsa_prefill_backend": "klxdsa", - "nsa_decode_backend": "klxdsa", - "disable_flashinfer_autotune": false, - "mamba_backend": "triton", - "speculative_algorithm": "EAGLE", - "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", - "speculative_draft_model_revision": null, - "speculative_draft_load_format": null, - "speculative_num_steps": 3, - "speculative_eagle_topk": 1, - "speculative_num_draft_tokens": 4, - "speculative_accept_threshold_single": 1.0, - "speculative_accept_threshold_acc": 1.0, - "speculative_token_map": null, - "speculative_attention_mode": "prefill", - "speculative_draft_attention_backend": null, - "speculative_moe_runner_backend": "auto", - "speculative_moe_a2a_backend": null, - "speculative_draft_model_quantization": "w8a8_int8", - "speculative_ngram_min_bfs_breadth": 1, - "speculative_ngram_max_bfs_breadth": 10, - "speculative_ngram_match_type": "BFS", - "speculative_ngram_max_trie_depth": 18, - "speculative_ngram_capacity": 10000000, - "enable_multi_layer_eagle": false, - "ep_size": 8, - "moe_a2a_backend": "none", - "moe_runner_backend": "auto", - "flashinfer_mxfp4_moe_precision": "default", - "enable_hisparse": false, - "enable_flashinfer_allreduce_fusion": false, - "enforce_disable_flashinfer_allreduce_fusion": false, - "enable_aiter_allreduce_fusion": false, - "deepep_mode": "auto", - "ep_num_redundant_experts": 0, - "ep_dispatch_algorithm": null, - "init_expert_location": "trivial", - "enable_eplb": false, - "eplb_algorithm": "auto", - "eplb_rebalance_num_iterations": 1000, - "eplb_rebalance_layers_per_chunk": null, - "eplb_min_rebalancing_utilization_threshold": 1.0, - "expert_distribution_recorder_mode": null, - "expert_distribution_recorder_buffer_size": 1000, - "enable_expert_distribution_metrics": false, - "deepep_config": null, - "moe_dense_tp_size": null, - "elastic_ep_backend": null, - "enable_elastic_expert_backup": false, - "mooncake_ib_device": null, - "max_mamba_cache_size": null, - "mamba_ssm_dtype": null, - "mamba_full_memory_ratio": 0.9, - "mamba_scheduler_strategy": "no_buffer", - "mamba_track_interval": 256, - "linear_attn_backend": "triton", - "linear_attn_decode_backend": null, - "linear_attn_prefill_backend": null, - "enable_hierarchical_cache": false, - "hicache_ratio": 2.0, - "hicache_size": 0, - "hicache_write_policy": "write_through", - "hicache_io_backend": "kernel", - "hicache_mem_layout": "page_first", - "hicache_storage_backend": null, - "hicache_storage_prefetch_policy": "best_effort", - "hicache_storage_backend_extra_config": null, - "hisparse_config": null, - "enable_lmcache": false, - "kt_weight_path": null, - "kt_method": "AMXINT4", - "kt_cpuinfer": null, - "kt_threadpool_count": 2, - "kt_num_gpu_experts": null, - "kt_max_deferred_experts_per_token": null, - "dllm_algorithm": null, - "dllm_algorithm_config": null, - "enable_double_sparsity": false, - "ds_channel_config_path": null, - "ds_heavy_channel_num": 32, - "ds_heavy_token_num": 256, - "ds_heavy_channel_type": "qk", - "ds_sparse_decode_threshold": 4096, - "cpu_offload_gb": 0, - "offload_group_size": -1, - "offload_num_in_group": 1, - "offload_prefetch_step": 1, - "offload_mode": "cpu", - "multi_item_scoring_delimiter": null, - "disable_radix_cache": false, - "cuda_graph_max_bs": 32, - "cuda_graph_bs": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 10, - 12, - 14, - 16, - 18, - 20, - 22, - 24, - 26, - 28, - 30, - 32 - ], - "disable_cuda_graph": false, - "disable_cuda_graph_padding": false, - "enable_profile_cuda_graph": false, - "enable_cudagraph_gc": false, - "enable_layerwise_nvtx_marker": false, - "enable_nccl_nvls": false, - "enable_symm_mem": false, - "disable_flashinfer_cutlass_moe_fp4_allgather": false, - "enable_tokenizer_batch_encode": false, - "disable_tokenizer_batch_decode": false, - "disable_outlines_disk_cache": false, - "disable_custom_all_reduce": true, - "enable_mscclpp": false, - "enable_torch_symm_mem": false, - "pre_warm_nccl": false, - "disable_overlap_schedule": false, - "enable_mixed_chunk": false, - "enable_dp_attention": false, - "enable_dp_lm_head": false, - "enable_two_batch_overlap": false, - "enable_single_batch_overlap": false, - "tbo_token_distribution_threshold": 0.48, - "enable_torch_compile": false, - "disable_piecewise_cuda_graph": true, - "enforce_piecewise_cuda_graph": false, - "enable_torch_compile_debug_mode": false, - "torch_compile_max_bs": 32, - "piecewise_cuda_graph_max_tokens": 8192, - "piecewise_cuda_graph_tokens": [ - 4, - 8, - 12, - 16, - 20, - 24, - 28, - 32, - 48, - 64, - 80, - 96, - 112, - 128, - 144, - 160, - 176, - 192, - 208, - 224, - 240, - 256, - 288, - 320, - 352, - 384, - 416, - 448, - 480, - 512, - 576, - 640, - 704, - 768, - 832, - 896, - 960, - 1024, - 1280, - 1536, - 1792, - 2048, - 2304, - 2560, - 2816, - 3072, - 3328, - 3584, - 3840, - 4096, - 4608, - 5120, - 5632, - 6144, - 6656, - 7168, - 7680, - 8192 - ], - "piecewise_cuda_graph_compiler": "eager", - "torchao_config": "", - "enable_nan_detection": false, - "enable_p2p_check": false, - "triton_attention_reduce_in_fp32": false, - "triton_attention_num_kv_splits": 8, - "triton_attention_split_tile_size": null, - "num_continuous_decode_steps": 1, - "delete_ckpt_after_loading": false, - "enable_memory_saver": false, - "enable_weights_cpu_backup": false, - "enable_draft_weights_cpu_backup": false, - "allow_auto_truncate": false, - "enable_custom_logit_processor": false, - "flashinfer_mla_disable_ragged": false, - "disable_shared_experts_fusion": true, - "disable_chunked_prefix_cache": false, - "disable_fast_image_processor": false, - "keep_mm_feature_on_device": false, - "enable_return_hidden_states": false, - "enable_return_routed_experts": false, - "enable_return_indexer_topk": false, - "scheduler_recv_interval": 1, - "numa_node": null, - "enable_deterministic_inference": false, - "rl_on_policy_target": null, - "enable_attn_tp_input_scattered": false, - "gc_threshold": null, - "enable_nsa_prefill_context_parallel": false, - "nsa_prefill_cp_mode": "in-seq-split", - "enable_fused_qk_norm_rope": false, - "enable_precise_embedding_interpolation": false, - "enable_fused_moe_sum_all_reduce": false, - "enable_prefill_context_parallel": false, - "prefill_cp_mode": "in-seq-split", - "enable_dynamic_batch_tokenizer": false, - "dynamic_batch_tokenizer_batch_size": 32, - "dynamic_batch_tokenizer_batch_timeout": 0.002, - "debug_tensor_dump_output_folder": null, - "debug_tensor_dump_layers": null, - "debug_tensor_dump_input_file": null, - "debug_tensor_dump_inject": false, - "disaggregation_mode": "null", - "disaggregation_transfer_backend": "mooncake", - "disaggregation_bootstrap_port": 8998, - "disaggregation_ib_device": null, - "disaggregation_decode_enable_offload_kvcache": false, - "num_reserved_decode_tokens": 512, - "disaggregation_decode_polling_interval": 1, - "encoder_only": false, - "language_only": false, - "encoder_transfer_backend": "zmq_to_scheduler", - "encoder_urls": [], - "enable_adaptive_dispatch_to_encoder": false, - "enable_over_encoding": false, - "enable_kv_mirror": false, - "custom_weight_loader": [], - "weight_loader_disable_mmap": false, - "remote_instance_weight_loader_seed_instance_ip": null, - "remote_instance_weight_loader_seed_instance_service_port": null, - "remote_instance_weight_loader_send_weights_group_ports": null, - "remote_instance_weight_loader_backend": "nccl", - "remote_instance_weight_loader_start_seed_via_transfer_engine": false, - "engine_info_bootstrap_port": 6789, - "modelexpress_config": null, - "enable_pdmux": false, - "pdmux_config_path": null, - "sm_group_num": 8, - "enable_broadcast_mm_inputs_process": false, - "enable_prefix_mm_cache": false, - "mm_enable_dp_encoder": false, - "mm_process_config": {}, - "limit_mm_data_per_request": null, - "enable_mm_global_cache": false, - "decrypted_config_file": null, - "decrypted_draft_config_file": null, - "forward_hooks": null, - "index_cache_topk_freq": 1, - "index_cache_pattern": null, - "status": "ready", - "max_total_num_tokens": 1593344, - "max_req_input_len": 1048570, - "internal_states": [ - { - "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", - "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", - "tokenizer_mode": "auto", - "tokenizer_worker_num": 1, - "skip_tokenizer_init": false, - "load_format": "auto", - "model_loader_extra_config": "{}", - "trust_remote_code": true, - "context_length": 1048576, - "is_embedding": false, - "enable_multimodal": null, - "revision": null, - "model_impl": "auto", - "host": "0.0.0.0", - "port": 30000, - "fastapi_root_path": "", - "grpc_mode": false, - "skip_server_warmup": false, - "warmups": null, - "nccl_port": null, - "checkpoint_engine_wait_weights_before_ready": false, - "ssl_keyfile": null, - "ssl_certfile": null, - "ssl_ca_certs": null, - "ssl_keyfile_password": null, - "enable_ssl_refresh": false, - "dtype": "auto", - "quantization": "w8a8_int8", - "quantization_param_path": null, - "kv_cache_dtype": "float16", - "enable_fp32_lm_head": false, - "modelopt_quant": null, - "modelopt_checkpoint_restore_path": null, - "modelopt_checkpoint_save_path": null, - "modelopt_export_path": null, - "quantize_and_serve": false, - "rl_quant_profile": null, - "mem_fraction_static": 0.8, - "max_running_requests": 64, - "max_queued_requests": null, - "max_total_tokens": null, - "chunked_prefill_size": 8192, - "enable_dynamic_chunking": false, - "max_prefill_tokens": 16384, - "prefill_max_requests": null, - "custom_max_context_len": null, - "schedule_policy": "fcfs", - "enable_priority_scheduling": false, - "disable_priority_preemption": false, - "default_priority_value": null, - "abort_on_priority_when_disabled": false, - "schedule_low_priority_values_first": false, - "priority_scheduling_preemption_threshold": 10, - "schedule_conservativeness": 1.0, - "page_size": 256, - "swa_full_tokens_ratio": 0.1, - "disable_hybrid_swa_memory": false, - "radix_eviction_policy": "lru", - "enable_prefill_delayer": false, - "prefill_delayer_max_delay_passes": 30, - "prefill_delayer_token_usage_low_watermark": null, - "prefill_delayer_forward_passes_buckets": null, - "prefill_delayer_wait_seconds_buckets": null, - "device": "cuda", - "tp_size": 8, - "pp_size": 1, - "pp_max_micro_batch_size": 64, - "pp_async_batch_depth": 0, - "stream_interval": 1, - "stream_response_default_include_usage": false, - "incremental_streaming_output": false, - "enable_streaming_session": false, - "random_seed": 228424360, - "constrained_json_whitespace_pattern": null, - "constrained_json_disable_any_whitespace": true, - "watchdog_timeout": 3000000.0, - "soft_watchdog_timeout": null, - "dist_timeout": null, - "download_dir": null, - "model_checksum": null, - "base_gpu_id": 0, - "gpu_id_step": 1, - "sleep_on_idle": false, - "use_ray": false, - "custom_sigquit_handler": null, - "log_level": "info", - "log_level_http": null, - "log_requests": false, - "log_requests_level": 2, - "log_requests_format": "text", - "log_requests_target": null, - "uvicorn_access_log_exclude_prefixes": [], - "crash_dump_folder": null, - "show_time_cost": false, - "enable_metrics": true, - "enable_mfu_metrics": false, - "enable_metrics_for_all_schedulers": false, - "tokenizer_metrics_custom_labels_header": "x-custom-labels", - "tokenizer_metrics_allowed_custom_labels": null, - "extra_metric_labels": null, - "bucket_time_to_first_token": null, - "bucket_inter_token_latency": null, - "bucket_e2e_request_latency": null, - "collect_tokens_histogram": false, - "prompt_tokens_buckets": null, - "generation_tokens_buckets": null, - "gc_warning_threshold_secs": 0.0, - "decode_log_interval": 40, - "enable_request_time_stats_logging": true, - "kv_events_config": null, - "enable_trace": false, - "otlp_traces_endpoint": "localhost:4317", - "export_metrics_to_file": false, - "export_metrics_to_file_dir": null, - "api_key": null, - "admin_api_key": null, - "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", - "weight_version": "default", - "chat_template": null, - "hf_chat_template_name": null, - "completion_template": null, - "file_storage_path": "sglang_storage", - "enable_cache_report": false, - "reasoning_parser": "deepseek-v4", - "tool_call_parser": "deepseekv4", - "tool_server": null, - "sampling_defaults": "model", - "dp_size": 1, - "load_balance_method": "round_robin", - "attn_cp_size": 1, - "moe_dp_size": 1, - "dist_init_addr": null, - "nnodes": 1, - "node_rank": 0, - "json_model_override_args": "{}", - "preferred_sampling_params": null, - "enable_lora": null, - "enable_lora_overlap_loading": null, - "max_lora_rank": null, - "lora_target_modules": null, - "lora_paths": null, - "max_loaded_loras": null, - "max_loras_per_batch": 8, - "lora_eviction_policy": "lru", - "lora_backend": "csgmv", - "max_lora_chunk_size": 16, - "experts_shared_outer_loras": null, - "attention_backend": "compressed", - "decode_attention_backend": "compressed", - "prefill_attention_backend": "compressed", - "sampling_backend": "flashinfer", - "grammar_backend": "xgrammar", - "mm_attention_backend": null, - "fp8_gemm_runner_backend": "auto", - "fp4_gemm_runner_backend": "auto", - "nsa_prefill_backend": "klxdsa", - "nsa_decode_backend": "klxdsa", - "disable_flashinfer_autotune": false, - "mamba_backend": "triton", - "speculative_algorithm": "EAGLE", - "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", - "speculative_draft_model_revision": null, - "speculative_draft_load_format": null, - "speculative_num_steps": 3, - "speculative_eagle_topk": 1, - "speculative_num_draft_tokens": 4, - "speculative_accept_threshold_single": 1.0, - "speculative_accept_threshold_acc": 1.0, - "speculative_token_map": null, - "speculative_attention_mode": "prefill", - "speculative_draft_attention_backend": null, - "speculative_moe_runner_backend": "auto", - "speculative_moe_a2a_backend": null, - "speculative_draft_model_quantization": "w8a8_int8", - "speculative_ngram_min_bfs_breadth": 1, - "speculative_ngram_max_bfs_breadth": 10, - "speculative_ngram_match_type": "BFS", - "speculative_ngram_max_trie_depth": 18, - "speculative_ngram_capacity": 10000000, - "enable_multi_layer_eagle": false, - "ep_size": 8, - "moe_a2a_backend": "none", - "moe_runner_backend": "auto", - "flashinfer_mxfp4_moe_precision": "default", - "enable_hisparse": false, - "enable_flashinfer_allreduce_fusion": false, - "enforce_disable_flashinfer_allreduce_fusion": false, - "enable_aiter_allreduce_fusion": false, - "deepep_mode": "auto", - "ep_num_redundant_experts": 0, - "ep_dispatch_algorithm": null, - "init_expert_location": "trivial", - "enable_eplb": false, - "eplb_algorithm": "auto", - "eplb_rebalance_num_iterations": 1000, - "eplb_rebalance_layers_per_chunk": null, - "eplb_min_rebalancing_utilization_threshold": 1.0, - "expert_distribution_recorder_mode": null, - "expert_distribution_recorder_buffer_size": 1000, - "enable_expert_distribution_metrics": false, - "deepep_config": null, - "moe_dense_tp_size": null, - "elastic_ep_backend": null, - "enable_elastic_expert_backup": false, - "mooncake_ib_device": null, - "max_mamba_cache_size": null, - "mamba_ssm_dtype": null, - "mamba_full_memory_ratio": 0.9, - "mamba_scheduler_strategy": "no_buffer", - "mamba_track_interval": 256, - "linear_attn_backend": "triton", - "linear_attn_decode_backend": null, - "linear_attn_prefill_backend": null, - "enable_hierarchical_cache": false, - "hicache_ratio": 2.0, - "hicache_size": 0, - "hicache_write_policy": "write_through", - "hicache_io_backend": "kernel", - "hicache_mem_layout": "page_first", - "hicache_storage_backend": null, - "hicache_storage_prefetch_policy": "best_effort", - "hicache_storage_backend_extra_config": null, - "hisparse_config": null, - "enable_lmcache": false, - "kt_weight_path": null, - "kt_method": "AMXINT4", - "kt_cpuinfer": null, - "kt_threadpool_count": 2, - "kt_num_gpu_experts": null, - "kt_max_deferred_experts_per_token": null, - "dllm_algorithm": null, - "dllm_algorithm_config": null, - "enable_double_sparsity": false, - "ds_channel_config_path": null, - "ds_heavy_channel_num": 32, - "ds_heavy_token_num": 256, - "ds_heavy_channel_type": "qk", - "ds_sparse_decode_threshold": 4096, - "cpu_offload_gb": 0, - "offload_group_size": -1, - "offload_num_in_group": 1, - "offload_prefetch_step": 1, - "offload_mode": "cpu", - "multi_item_scoring_delimiter": null, - "disable_radix_cache": false, - "cuda_graph_max_bs": 32, - "cuda_graph_bs": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 10, - 12, - 14, - 16, - 18, - 20, - 22, - 24, - 26, - 28, - 30, - 32 - ], - "disable_cuda_graph": false, - "disable_cuda_graph_padding": false, - "enable_profile_cuda_graph": false, - "enable_cudagraph_gc": false, - "enable_layerwise_nvtx_marker": false, - "enable_nccl_nvls": false, - "enable_symm_mem": false, - "disable_flashinfer_cutlass_moe_fp4_allgather": false, - "enable_tokenizer_batch_encode": false, - "disable_tokenizer_batch_decode": false, - "disable_outlines_disk_cache": false, - "disable_custom_all_reduce": true, - "enable_mscclpp": false, - "enable_torch_symm_mem": false, - "pre_warm_nccl": false, - "disable_overlap_schedule": false, - "enable_mixed_chunk": false, - "enable_dp_attention": false, - "enable_dp_lm_head": false, - "enable_two_batch_overlap": false, - "enable_single_batch_overlap": false, - "tbo_token_distribution_threshold": 0.48, - "enable_torch_compile": false, - "disable_piecewise_cuda_graph": true, - "enforce_piecewise_cuda_graph": false, - "enable_torch_compile_debug_mode": false, - "torch_compile_max_bs": 32, - "piecewise_cuda_graph_max_tokens": 8192, - "piecewise_cuda_graph_tokens": [ - 4, - 8, - 12, - 16, - 20, - 24, - 28, - 32, - 48, - 64, - 80, - 96, - 112, - 128, - 144, - 160, - 176, - 192, - 208, - 224, - 240, - 256, - 288, - 320, - 352, - 384, - 416, - 448, - 480, - 512, - 576, - 640, - 704, - 768, - 832, - 896, - 960, - 1024, - 1280, - 1536, - 1792, - 2048, - 2304, - 2560, - 2816, - 3072, - 3328, - 3584, - 3840, - 4096, - 4608, - 5120, - 5632, - 6144, - 6656, - 7168, - 7680, - 8192 - ], - "piecewise_cuda_graph_compiler": "eager", - "torchao_config": "", - "enable_nan_detection": false, - "enable_p2p_check": false, - "triton_attention_reduce_in_fp32": false, - "triton_attention_num_kv_splits": 8, - "triton_attention_split_tile_size": null, - "num_continuous_decode_steps": 1, - "delete_ckpt_after_loading": false, - "enable_memory_saver": false, - "enable_weights_cpu_backup": false, - "enable_draft_weights_cpu_backup": false, - "allow_auto_truncate": false, - "enable_custom_logit_processor": false, - "flashinfer_mla_disable_ragged": false, - "disable_shared_experts_fusion": true, - "disable_chunked_prefix_cache": true, - "disable_fast_image_processor": false, - "keep_mm_feature_on_device": false, - "enable_return_hidden_states": false, - "enable_return_routed_experts": false, - "enable_return_indexer_topk": false, - "scheduler_recv_interval": 1, - "numa_node": null, - "enable_deterministic_inference": false, - "rl_on_policy_target": null, - "enable_attn_tp_input_scattered": false, - "gc_threshold": null, - "enable_nsa_prefill_context_parallel": false, - "nsa_prefill_cp_mode": "in-seq-split", - "enable_fused_qk_norm_rope": false, - "enable_precise_embedding_interpolation": false, - "enable_fused_moe_sum_all_reduce": false, - "enable_prefill_context_parallel": false, - "prefill_cp_mode": "in-seq-split", - "enable_dynamic_batch_tokenizer": false, - "dynamic_batch_tokenizer_batch_size": 32, - "dynamic_batch_tokenizer_batch_timeout": 0.002, - "debug_tensor_dump_output_folder": null, - "debug_tensor_dump_layers": null, - "debug_tensor_dump_input_file": null, - "debug_tensor_dump_inject": false, - "disaggregation_mode": "null", - "disaggregation_transfer_backend": "mooncake", - "disaggregation_bootstrap_port": 8998, - "disaggregation_ib_device": null, - "disaggregation_decode_enable_offload_kvcache": false, - "num_reserved_decode_tokens": 512, - "disaggregation_decode_polling_interval": 1, - "encoder_only": false, - "language_only": false, - "encoder_transfer_backend": "zmq_to_scheduler", - "encoder_urls": [], - "enable_adaptive_dispatch_to_encoder": false, - "enable_over_encoding": false, - "enable_kv_mirror": false, - "custom_weight_loader": [], - "weight_loader_disable_mmap": false, - "remote_instance_weight_loader_seed_instance_ip": null, - "remote_instance_weight_loader_seed_instance_service_port": null, - "remote_instance_weight_loader_send_weights_group_ports": null, - "remote_instance_weight_loader_backend": "nccl", - "remote_instance_weight_loader_start_seed_via_transfer_engine": false, - "engine_info_bootstrap_port": 6789, - "modelexpress_config": null, - "enable_pdmux": false, - "pdmux_config_path": null, - "sm_group_num": 8, - "enable_broadcast_mm_inputs_process": false, - "enable_prefix_mm_cache": false, - "mm_enable_dp_encoder": false, - "mm_process_config": {}, - "limit_mm_data_per_request": null, - "enable_mm_global_cache": false, - "decrypted_config_file": null, - "decrypted_draft_config_file": null, - "forward_hooks": null, - "index_cache_topk_freq": 1, - "index_cache_pattern": null, - "use_mla_backend": false, - "_mx_config_cache": {}, - "_draft_pool_config": { - "full_max_total_num_tokens": 1593344, - "swa_max_total_num_tokens": 159232 - }, - "last_gen_throughput": 304.3209280340091, - "memory_usage": { - "weight": 37.94, - "kvcache": 0, - "token_capacity": 1593344, - "graph": 1.2 - }, - "effective_max_running_requests_per_dp": 64, - "avg_spec_accept_length": 2.8933333333333335 - } - ], - "version": "0.5.10+ee6948cf8" - }, - "duration": 5.34541754698148, - "completed": 10, - "total_input_tokens": 2517, - "total_input_text_tokens": 2517, - "total_input_vision_tokens": 0, - "total_output_tokens": 1404, - "total_output_tokens_retokenized": 1404, - "request_throughput": 1.870761247762007, - "input_throughput": 470.8706060616972, - "output_throughput": 262.6548791857858, - "total_throughput": 733.525485247483, - "mean_e2e_latency_ms": 3372.781996097183, - "median_e2e_latency_ms": 3498.675397480838, - "std_e2e_latency_ms": 1426.9563106485741, - "p90_e2e_latency_ms": 5125.400093087228, - "p99_e2e_latency_ms": 5314.590020017349, - "mean_ttft_ms": 478.8548628857825, - "median_ttft_ms": 506.4399079710711, - "std_ttft_ms": 82.89120389713447, - "p99_ttft_ms": 507.2712521097855, - "mean_tpot_ms": 22.334078483797782, - "median_tpot_ms": 21.948128755653826, - "std_tpot_ms": 3.1100348959353536, - "p99_tpot_ms": 29.154062995221466, - "mean_itl_ms": 20.75981864563044, - "median_itl_ms": 18.560815330905218, - "std_itl_ms": 11.544034313142475, - "p95_itl_ms": 30.416983005125076, - "p99_itl_ms": 76.09861597185954, - "concurrency": 6.309669855447998, - "accept_length": 2.8933333333333335, - "max_output_tokens_per_s": 396.0, - "max_concurrent_requests": 10, - "input_lens": [ - 38, - 236, - 397, - 73, - 256, - 394, - 204, - 134, - 336, - 449 - ], - "output_lens": [ - 145, - 130, - 205, - 72, - 238, - 253, - 135, - 26, - 179, - 21 - ], - "ttfts": [ - 0.23018694896018133, - 0.5072830729768611, - 0.5071517300093547, - 0.506980306992773, - 0.5067555939895101, - 0.5065123789827339, - 0.5063674369594082, - 0.5061119620222598, - 0.5057157859555446, - 0.5054834120091982 - ], - "itls": [ - [ - 0.1265941219899105, - 0.1265941219899105, - 0.1265941219899105, - 0.1265941219899105, - 0.03003738750703633, - 0.03003738750703633, - 0.020075495665272076, - 0.020075495665272076, - 0.020075495665272076, - 0.020034517995857943, - 0.020034517995857943, - 0.020034517995857943, - 0.020025947674488027, - 0.020025947674488027, - 0.020025947674488027, - 0.01982437400147319, - 0.01982437400147319, - 0.01982437400147319, - 0.030580375489080325, - 0.030580375489080325, - 0.015474200758035295, - 0.015474200758035295, - 0.015474200758035295, - 0.015474200758035295, - 0.02975128698744811, - 0.02975128698744811, - 0.018891388666816056, - 0.018891388666816056, - 0.018891388666816056, - 0.030985667331454653, - 0.030985667331454653, - 0.030985667331454653, - 0.01450748200295493, - 0.01450748200295493, - 0.01450748200295493, - 0.01450748200295493, - 0.02843167950049974, - 0.02843167950049974, - 0.019097025661418836, - 0.019097025661418836, - 0.019097025661418836, - 0.019033391009240102, - 0.019033391009240102, - 0.019033391009240102, - 0.02840644898242317, - 0.02840644898242317, - 0.0287785355176311, - 0.0287785355176311, - 0.01427662099013105, - 0.01427662099013105, - 0.01427662099013105, - 0.01427662099013105, - 0.028972148516913876, - 0.028972148516913876, - 0.0198178769981799, - 0.0198178769981799, - 0.0198178769981799, - 0.019058960994395118, - 0.019058960994395118, - 0.019058960994395118, - 0.014342486509121954, - 0.014342486509121954, - 0.014342486509121954, - 0.014342486509121954, - 0.028646462480537593, - 0.028646462480537593, - 0.01456742275331635, - 0.01456742275331635, - 0.01456742275331635, - 0.01456742275331635, - 0.01943501700103904, - 0.01943501700103904, - 0.01943501700103904, - 0.02791283550322987, - 0.02791283550322987, - 0.013792212252155878, - 0.013792212252155878, - 0.013792212252155878, - 0.013792212252155878, - 0.018324121988068025, - 0.018324121988068025, - 0.018324121988068025, - 0.018392249670190115, - 0.018392249670190115, - 0.018392249670190115, - 0.018568886327557266, - 0.018568886327557266, - 0.018568886327557266, - 0.018351545673795044, - 0.018351545673795044, - 0.018351545673795044, - 0.02752352401148528, - 0.02752352401148528, - 0.013863828251487575, - 0.013863828251487575, - 0.013863828251487575, - 0.013863828251487575, - 0.018301511649042368, - 0.018301511649042368, - 0.018301511649042368, - 0.027680501516442746, - 0.027680501516442746, - 0.01841827033786103, - 0.01841827033786103, - 0.01841827033786103, - 0.01374593649234157, - 0.01374593649234157, - 0.01374593649234157, - 0.01374593649234157, - 0.018055193999316543, - 0.018055193999316543, - 0.018055193999316543, - 0.014001160496263765, - 0.014001160496263765, - 0.014001160496263765, - 0.014001160496263765, - 0.018516544679490227, - 0.018516544679490227, - 0.018516544679490227, - 0.018641953337161492, - 0.018641953337161492, - 0.018641953337161492, - 0.01858119231959184, - 0.01858119231959184, - 0.01858119231959184, - 0.018560895676879834, - 0.018560895676879834, - 0.018560895676879834, - 0.027825927507365122, - 0.027825927507365122, - 0.027690790477208793, - 0.027690790477208793, - 0.014033740997547284, - 0.014033740997547284, - 0.014033740997547284, - 0.014033740997547284, - 0.028556485020089895, - 0.028556485020089895, - 0.018292231000183772, - 0.018292231000183772, - 0.018292231000183772, - 0.03276129499621069, - 0.03276129499621069, - 0.03276129499621069 - ], - [ - 0.114176959003089, - 0.114176959003089, - 0.02001935833444198, - 0.02001935833444198, - 0.02001935833444198, - 0.020075112329019856, - 0.020075112329019856, - 0.020075112329019856, - 0.02003466134192422, - 0.02003466134192422, - 0.02003466134192422, - 0.020026450996131945, - 0.020026450996131945, - 0.020026450996131945, - 0.02973513101460412, - 0.02973513101460412, - 0.030412352993153036, - 0.030412352993153036, - 0.015535941754933447, - 0.015535941754933447, - 0.015535941754933447, - 0.015535941754933447, - 0.02982658799737692, - 0.02982658799737692, - 0.018890631986626733, - 0.018890631986626733, - 0.018890631986626733, - 0.03098562400555238, - 0.03098562400555238, - 0.03098562400555238, - 0.019343592672764014, - 0.019343592672764014, - 0.019343592672764014, - 0.018950106323851895, - 0.018950106323851895, - 0.018950106323851895, - 0.019099269004072994, - 0.019099269004072994, - 0.019099269004072994, - 0.019034251008027542, - 0.019034251008027542, - 0.019034251008027542, - 0.01420391199644655, - 0.01420391199644655, - 0.01420391199644655, - 0.01420391199644655, - 0.01439423524425365, - 0.01439423524425365, - 0.01439423524425365, - 0.01439423524425365, - 0.028566046996274963, - 0.028566046996274963, - 0.01930476234216864, - 0.01930476234216864, - 0.01930476234216864, - 0.019809113660206396, - 0.019809113660206396, - 0.019809113660206396, - 0.014296345761977136, - 0.014296345761977136, - 0.014296345761977136, - 0.014296345761977136, - 0.02867897297255695, - 0.02867897297255695, - 0.019099251677592594, - 0.019099251677592594, - 0.019099251677592594, - 0.019382666000941146, - 0.019382666000941146, - 0.019382666000941146, - 0.02924422750947997, - 0.02924422750947997, - 0.027913359983358532, - 0.027913359983358532, - 0.013784522248897702, - 0.013784522248897702, - 0.013784522248897702, - 0.013784522248897702, - 0.02746082251542248, - 0.02746082251542248, - 0.01840783333561073, - 0.01840783333561073, - 0.01840783333561073, - 0.01856821965581427, - 0.01856821965581427, - 0.01856821965581427, - 0.01835258233283336, - 0.01835258233283336, - 0.01835258233283336, - 0.018347706005442888, - 0.018347706005442888, - 0.018347706005442888, - 0.018484461004845798, - 0.018484461004845798, - 0.018484461004845798, - 0.018302411655895412, - 0.018302411655895412, - 0.018302411655895412, - 0.027679786493536085, - 0.027679786493536085, - 0.027627180505078286, - 0.027627180505078286, - 0.01375247175747063, - 0.01375247175747063, - 0.01375247175747063, - 0.01375247175747063, - 0.0270681754918769, - 0.0270681754918769, - 0.01866779732517898, - 0.01866779732517898, - 0.01866779732517898, - 0.018516544679490227, - 0.018516544679490227, - 0.018516544679490227, - 0.013980599993374199, - 0.013980599993374199, - 0.013980599993374199, - 0.013980599993374199, - 0.027871913509443402, - 0.027871913509443402, - 0.018560129004375387, - 0.018560129004375387, - 0.018560129004375387, - 0.018560815330905218, - 0.018560815330905218, - 0.018560815330905218, - 0.027690300485119224, - 0.027690300485119224, - 0.05595995101612061 - ], - [ - 0.07610034267418087, - 0.07610034267418087, - 0.07610034267418087, - 0.02001753800626223, - 0.02001753800626223, - 0.02001753800626223, - 0.020074322334645938, - 0.020074322334645938, - 0.020074322334645938, - 0.015025673492345959, - 0.015025673492345959, - 0.015025673492345959, - 0.015025673492345959, - 0.03003843649639748, - 0.03003843649639748, - 0.01982440067998444, - 0.01982440067998444, - 0.01982440067998444, - 0.02034523299274345, - 0.02034523299274345, - 0.02034523299274345, - 0.030977007001638412, - 0.030977007001638412, - 0.029805967991705984, - 0.029805967991705984, - 0.014167338755214587, - 0.014167338755214587, - 0.014167338755214587, - 0.014167338755214587, - 0.046478810982080176, - 0.046478810982080176, - 0.01934337633429095, - 0.01934337633429095, - 0.01934337633429095, - 0.0189525830016161, - 0.0189525830016161, - 0.0189525830016161, - 0.014323239010991529, - 0.014323239010991529, - 0.014323239010991529, - 0.014323239010991529, - 0.028550161485327408, - 0.028550161485327408, - 0.018938299675937742, - 0.018938299675937742, - 0.018938299675937742, - 0.019176949999139953, - 0.019176949999139953, - 0.019176949999139953, - 0.028566726978169754, - 0.028566726978169754, - 0.028956993512110785, - 0.028956993512110785, - 0.014857275251415558, - 0.014857275251415558, - 0.014857275251415558, - 0.014857275251415558, - 0.028592451504664496, - 0.028592451504664496, - 0.01912001532036811, - 0.01912001532036811, - 0.01912001532036811, - 0.019098964675019186, - 0.019098964675019186, - 0.019098964675019186, - 0.014537554743583314, - 0.014537554743583314, - 0.014537554743583314, - 0.014537554743583314, - 0.029245732497656718, - 0.029245732497656718, - 0.018609130017769832, - 0.018609130017769832, - 0.018609130017769832, - 0.018378569666917127, - 0.018378569666917127, - 0.018378569666917127, - 0.027461227495223284, - 0.027461227495223284, - 0.013806014991132542, - 0.013806014991132542, - 0.013806014991132542, - 0.013806014991132542, - 0.013926397252362221, - 0.013926397252362221, - 0.013926397252362221, - 0.013926397252362221, - 0.01835247567699601, - 0.01835247567699601, - 0.01835247567699601, - 0.01834764933058371, - 0.01834764933058371, - 0.01834764933058371, - 0.013863673244486563, - 0.013863673244486563, - 0.013863673244486563, - 0.013863673244486563, - 0.027453012502519414, - 0.027453012502519414, - 0.018453627669562895, - 0.018453627669562895, - 0.018453627669562895, - 0.018418410327285528, - 0.018418410327285528, - 0.018418410327285528, - 0.027505588514031842, - 0.027505588514031842, - 0.013533967750845477, - 0.013533967750845477, - 0.013533967750845477, - 0.013533967750845477, - 0.014000577997649089, - 0.014000577997649089, - 0.014000577997649089, - 0.014000577997649089, - 0.018516601334946852, - 0.018516601334946852, - 0.018516601334946852, - 0.018641606671735644, - 0.018641606671735644, - 0.018641606671735644, - 0.013935844239313155, - 0.013935844239313155, - 0.013935844239313155, - 0.013935844239313155, - 0.02784075849922374, - 0.02784075849922374, - 0.018561025344145794, - 0.018561025344145794, - 0.018561025344145794, - 0.01846017032706489, - 0.01846017032706489, - 0.01846017032706489, - 0.02801839599851519, - 0.02801839599851519, - 0.014315253254608251, - 0.014315253254608251, - 0.014315253254608251, - 0.014315253254608251, - 0.013719310751184821, - 0.013719310751184821, - 0.013719310751184821, - 0.013719310751184821, - 0.032736671001960836, - 0.032736671001960836, - 0.032736671001960836, - 0.01810533132326479, - 0.01810533132326479, - 0.01810533132326479, - 0.025817634508712217, - 0.025817634508712217, - 0.017274240672122687, - 0.017274240672122687, - 0.017274240672122687, - 0.012808780491468497, - 0.012808780491468497, - 0.012808780491468497, - 0.012808780491468497, - 0.017103748007987935, - 0.017103748007987935, - 0.017103748007987935, - 0.0260774539783597, - 0.0260774539783597, - 0.012770477012963966, - 0.012770477012963966, - 0.012770477012963966, - 0.012770477012963966, - 0.012633724749321118, - 0.012633724749321118, - 0.012633724749321118, - 0.012633724749321118, - 0.016969308664556593, - 0.016969308664556593, - 0.016969308664556593, - 0.017546184652019292, - 0.017546184652019292, - 0.017546184652019292, - 0.02627176101668738, - 0.02627176101668738, - 0.01677496867099156, - 0.01677496867099156, - 0.01677496867099156, - 0.012476857242290862, - 0.012476857242290862, - 0.012476857242290862, - 0.012476857242290862, - 0.016706993997407455, - 0.016706993997407455, - 0.016706993997407455, - 0.02511487199808471, - 0.02511487199808471, - 0.012620112000149675, - 0.012620112000149675, - 0.012620112000149675, - 0.012620112000149675, - 0.013204801754909568, - 0.013204801754909568, - 0.013204801754909568, - 0.013204801754909568, - 0.016931711327439796, - 0.016931711327439796, - 0.016931711327439796, - 0.025843739014817402, - 0.025843739014817402 - ], - [ - 0.07609820265012483, - 0.07609820265012483, - 0.07609820265012483, - 0.020024441337833803, - 0.020024441337833803, - 0.020024441337833803, - 0.015056459509651177, - 0.015056459509651177, - 0.015056459509651177, - 0.015056459509651177, - 0.03005173648125492, - 0.03005173648125492, - 0.015019418249721639, - 0.015019418249721639, - 0.015019418249721639, - 0.015019418249721639, - 0.019824344334968675, - 0.019824344334968675, - 0.019824344334968675, - 0.020387480326462537, - 0.020387480326462537, - 0.020387480326462537, - 0.020631657680496573, - 0.020631657680496573, - 0.020631657680496573, - 0.01487621849810239, - 0.01487621849810239, - 0.01487621849810239, - 0.01487621849810239, - 0.028335707494989038, - 0.028335707494989038, - 0.023239640504471026, - 0.023239640504471026, - 0.023239640504471026, - 0.023239640504471026, - 0.0193434563310196, - 0.0193434563310196, - 0.0193434563310196, - 0.028431184502551332, - 0.028431184502551332, - 0.014322734248708002, - 0.014322734248708002, - 0.014322734248708002, - 0.014322734248708002, - 0.01903337399320056, - 0.01903337399320056, - 0.01903337399320056, - 0.01893743300267185, - 0.01893743300267185, - 0.01893743300267185, - 0.01917593334413444, - 0.01917593334413444, - 0.01917593334413444, - 0.028568201989401132, - 0.028568201989401132, - 0.01930442900629714, - 0.01930442900629714, - 0.01930442900629714, - 0.014869755497784354, - 0.014869755497784354, - 0.014869755497784354, - 0.014869755497784354, - 0.028589651483343914, - 0.028589651483343914, - 0.019121012010145932, - 0.019121012010145932, - 0.019121012010145932, - 0.01909933132507528, - 0.01909933132507528, - 0.01909933132507528, - 0.0582002800074406 - ], - [ - 0.07609824265819043, - 0.07609824265819043, - 0.07609824265819043, - 0.015011793249868788, - 0.015011793249868788, - 0.015011793249868788, - 0.015011793249868788, - 0.020074732679252822, - 0.020074732679252822, - 0.020074732679252822, - 0.015025845990749076, - 0.015025845990749076, - 0.015025845990749076, - 0.015025845990749076, - 0.030038921511732042, - 0.030038921511732042, - 0.019824267326233286, - 0.019824267326233286, - 0.019824267326233286, - 0.020395867332505684, - 0.020395867332505684, - 0.020395867332505684, - 0.03094582151970826, - 0.03094582151970826, - 0.014877470748615451, - 0.014877470748615451, - 0.014877470748615451, - 0.014877470748615451, - 0.014167653993354179, - 0.014167653993354179, - 0.014167653993354179, - 0.014167653993354179, - 0.030986197331609826, - 0.030986197331609826, - 0.030986197331609826, - 0.01934319967404008, - 0.01934319967404008, - 0.01934319967404008, - 0.014215631992556155, - 0.014215631992556155, - 0.014215631992556155, - 0.014215631992556155, - 0.02864447850151919, - 0.02864447850151919, - 0.01903398767656957, - 0.01903398767656957, - 0.01903398767656957, - 0.018937196330322575, - 0.018937196330322575, - 0.018937196330322575, - 0.028763925016392022, - 0.028763925016392022, - 0.02856709697516635, - 0.02856709697516635, - 0.014478914003120735, - 0.014478914003120735, - 0.014478914003120735, - 0.014478914003120735, - 0.029714685515500605, - 0.029714685515500605, - 0.019061427660441648, - 0.019061427660441648, - 0.019061427660441648, - 0.019120605332621683, - 0.019120605332621683, - 0.019120605332621683, - 0.014324548756121658, - 0.014324548756121658, - 0.014324548756121658, - 0.014324548756121658, - 0.0290751994762104, - 0.0290751994762104, - 0.019497701665386558, - 0.019497701665386558, - 0.019497701665386558, - 0.01860897334214921, - 0.01860897334214921, - 0.01860897334214921, - 0.02756879449589178, - 0.02756879449589178, - 0.02748415298992768, - 0.02748415298992768, - 0.01379515475127846, - 0.01379515475127846, - 0.01379515475127846, - 0.01379515475127846, - 0.02785251452587545, - 0.02785251452587545, - 0.018352062324993312, - 0.018352062324993312, - 0.018352062324993312, - 0.01834822599388038, - 0.01834822599388038, - 0.01834822599388038, - 0.01386378325696569, - 0.01386378325696569, - 0.01386378325696569, - 0.01386378325696569, - 0.02745306250290014, - 0.02745306250290014, - 0.018453344334072124, - 0.018453344334072124, - 0.018453344334072124, - 0.018418603654329974, - 0.018418603654329974, - 0.018418603654329974, - 0.02750637350254692, - 0.02750637350254692, - 0.027067835500929505, - 0.027067835500929505, - 0.014000943003338762, - 0.014000943003338762, - 0.014000943003338762, - 0.014000943003338762, - 0.027774896996561438, - 0.027774896996561438, - 0.01864148332970217, - 0.01864148332970217, - 0.01864148332970217, - 0.018581362344169367, - 0.018581362344169367, - 0.018581362344169367, - 0.013920626748586074, - 0.013920626748586074, - 0.013920626748586074, - 0.013920626748586074, - 0.027841037983307615, - 0.027841037983307615, - 0.018460453681958217, - 0.018460453681958217, - 0.018460453681958217, - 0.018680113988618057, - 0.018680113988618057, - 0.018680113988618057, - 0.028647751518292353, - 0.028647751518292353, - 0.02743947648559697, - 0.02743947648559697, - 0.024545218504499644, - 0.024545218504499644, - 0.024545218504499644, - 0.024545218504499644, - 0.027157786476891488, - 0.027157786476891488, - 0.017211266347051907, - 0.017211266347051907, - 0.017211266347051907, - 0.017274483999547858, - 0.017274483999547858, - 0.017274483999547858, - 0.012808570492779836, - 0.012808570492779836, - 0.012808570492779836, - 0.012808570492779836, - 0.02565537701593712, - 0.02565537701593712, - 0.017431950002598267, - 0.017431950002598267, - 0.017431950002598267, - 0.016987925317759316, - 0.016987925317759316, - 0.016987925317759316, - 0.025259989517508075, - 0.025259989517508075, - 0.012736934004351497, - 0.012736934004351497, - 0.012736934004351497, - 0.012736934004351497, - 0.013151180988643318, - 0.013151180988643318, - 0.013151180988643318, - 0.013151180988643318, - 0.01752863434376195, - 0.01752863434376195, - 0.01752863434376195, - 0.01677370866915832, - 0.01677370866915832, - 0.01677370866915832, - 0.01247714224155061, - 0.01247714224155061, - 0.01247714224155061, - 0.01247714224155061, - 0.025057951017515734, - 0.025057951017515734, - 0.016742911325612415, - 0.016742911325612415, - 0.016742911325612415, - 0.016827172657940537, - 0.016827172657940537, - 0.016827172657940537, - 0.0176063990026402, - 0.0176063990026402, - 0.0176063990026402, - 0.01693138466604675, - 0.01693138466604675, - 0.01693138466604675, - 0.012955810001585633, - 0.012955810001585633, - 0.012955810001585633, - 0.012955810001585633, - 0.016913124340741586, - 0.016913124340741586, - 0.016913124340741586, - 0.015995322338615853, - 0.015995322338615853, - 0.015995322338615853, - 0.012007721743430011, - 0.012007721743430011, - 0.012007721743430011, - 0.012007721743430011, - 0.024051369488006458, - 0.024051369488006458, - 0.015974845349167783, - 0.015974845349167783, - 0.015974845349167783, - 0.015958434999144327, - 0.015958434999144327, - 0.015958434999144327, - 0.015792935659798484, - 0.015792935659798484, - 0.015792935659798484, - 0.01592183733979861, - 0.01592183733979861, - 0.01592183733979861, - 0.011986501252977178, - 0.011986501252977178, - 0.011986501252977178, - 0.011986501252977178, - 0.01585641598406558, - 0.01585641598406558, - 0.01585641598406558, - 0.015924710663966835, - 0.015924710663966835, - 0.015924710663966835, - 0.01634039101190865, - 0.01634039101190865, - 0.01634039101190865 - ], - [ - 0.057074277006904595, - 0.057074277006904595, - 0.057074277006904595, - 0.057074277006904595, - 0.030024631996639073, - 0.030024631996639073, - 0.020074642340963084, - 0.020074642340963084, - 0.020074642340963084, - 0.020034064655192196, - 0.020034064655192196, - 0.020034064655192196, - 0.015019408252555877, - 0.015019408252555877, - 0.015019408252555877, - 0.015019408252555877, - 0.02973640151321888, - 0.02973640151321888, - 0.020345932998073597, - 0.020345932998073597, - 0.020345932998073597, - 0.020652821326317888, - 0.020652821326317888, - 0.020652821326317888, - 0.014902361246640794, - 0.014902361246640794, - 0.014902361246640794, - 0.014902361246640794, - 0.014167511501000263, - 0.014167511501000263, - 0.014167511501000263, - 0.014167511501000263, - 0.04647939599817619, - 0.04647939599817619, - 0.019343009334988892, - 0.019343009334988892, - 0.019343009334988892, - 0.01895378966582939, - 0.01895378966582939, - 0.01895378966582939, - 0.014322384260594845, - 0.014322384260594845, - 0.014322384260594845, - 0.014322384260594845, - 0.02855006148456596, - 0.02855006148456596, - 0.018938185996375978, - 0.018938185996375978, - 0.018938185996375978, - 0.019176096674830962, - 0.019176096674830962, - 0.019176096674830962, - 0.019044518005102873, - 0.019044518005102873, - 0.019044518005102873, - 0.019305008987430483, - 0.019305008987430483, - 0.019305008987430483, - 0.014857510250294581, - 0.014857510250294581, - 0.014857510250294581, - 0.014857510250294581, - 0.0190613410086371, - 0.0190613410086371, - 0.0190613410086371, - 0.019120295338022213, - 0.019120295338022213, - 0.019120295338022213, - 0.0143242962367367, - 0.0143242962367367, - 0.0143242962367367, - 0.0143242962367367, - 0.029075569502310827, - 0.029075569502310827, - 0.019497531349770725, - 0.019497531349770725, - 0.019497531349770725, - 0.018608823651447892, - 0.018608823651447892, - 0.018608823651447892, - 0.018379226346344996, - 0.018379226346344996, - 0.018379226346344996, - 0.018320148325680446, - 0.018320148325680446, - 0.018320148325680446, - 0.01379665250715334, - 0.01379665250715334, - 0.01379665250715334, - 0.01379665250715334, - 0.01856846265339603, - 0.01856846265339603, - 0.01856846265339603, - 0.018352199345827103, - 0.018352199345827103, - 0.018352199345827103, - 0.018347878998611122, - 0.018347878998611122, - 0.018347878998611122, - 0.02772726200055331, - 0.02772726200055331, - 0.013726471239351667, - 0.013726471239351667, - 0.013726471239351667, - 0.013726471239351667, - 0.018453674002860982, - 0.018453674002860982, - 0.018453674002860982, - 0.027627600502455607, - 0.027627600502455607, - 0.013753056759014726, - 0.013753056759014726, - 0.013753056759014726, - 0.013753056759014726, - 0.013533932986319996, - 0.013533932986319996, - 0.013533932986319996, - 0.013533932986319996, - 0.018667823673846822, - 0.018667823673846822, - 0.018667823673846822, - 0.01851649465970695, - 0.01851649465970695, - 0.01851649465970695, - 0.027962180523900315, - 0.027962180523900315, - 0.01858143232918034, - 0.01858143232918034, - 0.01858143232918034, - 0.01392033399315551, - 0.01392033399315551, - 0.01392033399315551, - 0.01392033399315551, - 0.01856105534049372, - 0.01856105534049372, - 0.01856105534049372, - 0.027690295508364215, - 0.027690295508364215, - 0.014009808248374611, - 0.014009808248374611, - 0.014009808248374611, - 0.014009808248374611, - 0.014324050745926797, - 0.014324050745926797, - 0.014324050745926797, - 0.014324050745926797, - 0.018292734341230243, - 0.018292734341230243, - 0.018292734341230243, - 0.03272628099269544, - 0.03272628099269544, - 0.03272628099269544, - 0.027157796488609165, - 0.027157796488609165, - 0.017211413010954857, - 0.017211413010954857, - 0.017211413010954857, - 0.012955822996445931, - 0.012955822996445931, - 0.012955822996445931, - 0.012955822996445931, - 0.017078191003141303, - 0.017078191003141303, - 0.017078191003141303, - 0.02565545149263926, - 0.02565545149263926, - 0.026133759994991124, - 0.026133759994991124, - 0.012745351748890243, - 0.012745351748890243, - 0.012745351748890243, - 0.012745351748890243, - 0.02526378951733932, - 0.02526378951733932, - 0.016981848670790594, - 0.016981848670790594, - 0.016981848670790594, - 0.01753506433063497, - 0.01753506433063497, - 0.01753506433063497, - 0.013146101002348587, - 0.013146101002348587, - 0.013146101002348587, - 0.013146101002348587, - 0.025147862499579787, - 0.025147862499579787, - 0.016644779650960118, - 0.016644779650960118, - 0.016644779650960118, - 0.01670563401421532, - 0.01670563401421532, - 0.01670563401421532, - 0.025114176998613402, - 0.025114176998613402, - 0.025240779505111277, - 0.025240779505111277, - 0.01760628231568262, - 0.01760628231568262, - 0.01760628231568262, - 0.016931691012966137, - 0.016931691012966137, - 0.016931691012966137, - 0.01727279700571671, - 0.01727279700571671, - 0.01727279700571671, - 0.016912257318229724, - 0.016912257318229724, - 0.016912257318229724, - 0.01598840901472916, - 0.01598840901472916, - 0.01598840901472916, - 0.016010235665210832, - 0.016010235665210832, - 0.016010235665210832, - 0.016034339651620638, - 0.016034339651620638, - 0.016034339651620638, - 0.015974988675831508, - 0.015974988675831508, - 0.015974988675831508, - 0.023938142490806058, - 0.023938142490806058, - 0.023689378518611193, - 0.023689378518611193, - 0.01592199399601668, - 0.01592199399601668, - 0.01592199399601668, - 0.01598192499174426, - 0.01598192499174426, - 0.01598192499174426, - 0.01586189299511413, - 0.01586189299511413, - 0.01586189299511413, - 0.015926023673576612, - 0.015926023673576612, - 0.015926023673576612, - 0.016315380674010765, - 0.016315380674010765, - 0.016315380674010765, - 0.017069103002237778, - 0.017069103002237778, - 0.017069103002237778, - 0.015235118994799754, - 0.015235118994799754, - 0.015235118994799754, - 0.015159054659307003, - 0.015159054659307003, - 0.015159054659307003, - 0.022672360995784402, - 0.022672360995784402, - 0.023076677520293742, - 0.023076677520293742 - ], - [ - 0.07599885431894411, - 0.07599885431894411, - 0.07599885431894411, - 0.020025841678337503, - 0.020025841678337503, - 0.020025841678337503, - 0.020075395664510626, - 0.020075395664510626, - 0.020075395664510626, - 0.020034617996619392, - 0.020034617996619392, - 0.020034617996619392, - 0.020026374336642522, - 0.020026374336642522, - 0.020026374336642522, - 0.019823340660271544, - 0.019823340660271544, - 0.019823340660271544, - 0.030416983005125076, - 0.030416983005125076, - 0.03106233850121498, - 0.03106233850121498, - 0.019885292005104322, - 0.019885292005104322, - 0.019885292005104322, - 0.018892501985343795, - 0.018892501985343795, - 0.018892501985343795, - 0.030985270665648084, - 0.030985270665648084, - 0.030985270665648084, - 0.019343586017688114, - 0.019343586017688114, - 0.019343586017688114, - 0.018950182983341318, - 0.018950182983341318, - 0.018950182983341318, - 0.019099209011377145, - 0.019099209011377145, - 0.019099209011377145, - 0.01903421765503784, - 0.01903421765503784, - 0.01903421765503784, - 0.018937779338254284, - 0.018937779338254284, - 0.018937779338254284, - 0.02875587999005802, - 0.02875587999005802, - 0.02856260701082647, - 0.02856260701082647, - 0.019304305334420253, - 0.019304305334420253, - 0.019304305334420253, - 0.019817157338062923, - 0.019817157338062923, - 0.019817157338062923, - 0.019061451001713674, - 0.019061451001713674, - 0.019061451001713674, - 0.019113485332733642, - 0.019113485332733642, - 0.019113485332733642, - 0.019104608334600925, - 0.019104608334600925, - 0.019104608334600925, - 0.02907446399331093, - 0.02907446399331093, - 0.014618368746596389, - 0.014618368746596389, - 0.014618368746596389, - 0.014618368746596389, - 0.018610139998296898, - 0.018610139998296898, - 0.018610139998296898, - 0.02757459500571713, - 0.02757459500571713, - 0.027460306999273598, - 0.027460306999273598, - 0.01380542025435716, - 0.01380542025435716, - 0.01380542025435716, - 0.01380542025435716, - 0.027853768988279626, - 0.027853768988279626, - 0.0183525126776658, - 0.0183525126776658, - 0.0183525126776658, - 0.018347032329378028, - 0.018347032329378028, - 0.018347032329378028, - 0.018484470997160923, - 0.018484470997160923, - 0.018484470997160923, - 0.01830341199335332, - 0.01830341199335332, - 0.01830341199335332, - 0.01845283400810634, - 0.01845283400810634, - 0.01845283400810634, - 0.018417870334815234, - 0.018417870334815234, - 0.018417870334815234, - 0.027498273499077186, - 0.027498273499077186, - 0.027074395504314452, - 0.027074395504314452, - 0.018667700661656756, - 0.018667700661656756, - 0.018667700661656756, - 0.01851662133897965, - 0.01851662133897965, - 0.01851662133897965, - 0.018641199994211394, - 0.018641199994211394, - 0.018641199994211394, - 0.018580482341349125, - 0.018580482341349125, - 0.018580482341349125, - 0.018559868990754087, - 0.018559868990754087, - 0.018559868990754087, - 0.018557828676421195, - 0.018557828676421195, - 0.018557828676421195, - 0.018460286994619917, - 0.018460286994619917, - 0.018460286994619917, - 0.018637193328080077, - 0.018637193328080077, - 0.018637193328080077, - 0.028849135007476434, - 0.028849135007476434, - 0.027445936488220468, - 0.027445936488220468 - ], - [ - 0.07609848600501816, - 0.07609848600501816, - 0.07609848600501816, - 0.02001590799773112, - 0.02001590799773112, - 0.02001590799773112, - 0.020074725985371817, - 0.020074725985371817, - 0.020074725985371817, - 0.015025570755824447, - 0.015025570755824447, - 0.015025570755824447, - 0.015025570755824447, - 0.0200260576675646, - 0.0200260576675646, - 0.0200260576675646, - 0.014868118247250095, - 0.014868118247250095, - 0.014868118247250095, - 0.014868118247250095, - 0.01529539775219746, - 0.01529539775219746, - 0.01529539775219746, - 0.01529539775219746, - 0.0618297120090574 - ], - [ - 0.11415219900663942, - 0.11415219900663942, - 0.020037391669272136, - 0.020037391669272136, - 0.020037391669272136, - 0.015055509502417408, - 0.015055509502417408, - 0.015055509502417408, - 0.015055509502417408, - 0.020034414328013856, - 0.020034414328013856, - 0.020034414328013856, - 0.020025807665660977, - 0.020025807665660977, - 0.020025807665660977, - 0.01982415100792423, - 0.01982415100792423, - 0.01982415100792423, - 0.030516334489220753, - 0.030516334489220753, - 0.015466665499843657, - 0.015466665499843657, - 0.015466665499843657, - 0.015466665499843657, - 0.019884118674478184, - 0.019884118674478184, - 0.019884118674478184, - 0.028335032984614372, - 0.028335032984614372, - 0.023239655507495627, - 0.023239655507495627, - 0.023239655507495627, - 0.023239655507495627, - 0.014507339743431658, - 0.014507339743431658, - 0.014507339743431658, - 0.014507339743431658, - 0.01895076932851225, - 0.01895076932851225, - 0.01895076932851225, - 0.019099789011913042, - 0.019099789011913042, - 0.019099789011913042, - 0.028550306480610743, - 0.028550306480610743, - 0.018938216341969866, - 0.018938216341969866, - 0.018938216341969866, - 0.019193043660682935, - 0.019193043660682935, - 0.019193043660682935, - 0.019044294662307948, - 0.019044294662307948, - 0.019044294662307948, - 0.019304565348041553, - 0.019304565348041553, - 0.019304565348041553, - 0.014856965237413533, - 0.014856965237413533, - 0.014856965237413533, - 0.014856965237413533, - 0.014296558263595216, - 0.014296558263595216, - 0.014296558263595216, - 0.014296558263595216, - 0.01911949533193062, - 0.01911949533193062, - 0.01911949533193062, - 0.019099241665874917, - 0.019099241665874917, - 0.019099241665874917, - 0.029074459482217208, - 0.029074459482217208, - 0.01949675833263124, - 0.01949675833263124, - 0.01949675833263124, - 0.013956615002825856, - 0.013956615002825856, - 0.013956615002825856, - 0.013956615002825856, - 0.018378923006821424, - 0.018378923006821424, - 0.018378923006821424, - 0.027460777491796762, - 0.027460777491796762, - 0.013805887501803227, - 0.013805887501803227, - 0.013805887501803227, - 0.013805887501803227, - 0.013926447005360387, - 0.013926447005360387, - 0.013926447005360387, - 0.013926447005360387, - 0.01835258931775267, - 0.01835258931775267, - 0.01835258931775267, - 0.018347795664643247, - 0.018347795664643247, - 0.018347795664643247, - 0.013863318512449041, - 0.013863318512449041, - 0.013863318512449041, - 0.013863318512449041, - 0.027453502494608983, - 0.027453502494608983, - 0.01845339099721362, - 0.01845339099721362, - 0.01845339099721362, - 0.018418139991505694, - 0.018418139991505694, - 0.018418139991505694, - 0.027505743521032855, - 0.027505743521032855, - 0.027068145980592817, - 0.027068145980592817, - 0.0186672803441373, - 0.0186672803441373, - 0.0186672803441373, - 0.018516397996184725, - 0.018516397996184725, - 0.018516397996184725, - 0.018641243669359635, - 0.018641243669359635, - 0.018641243669359635, - 0.018581482329561066, - 0.018581482329561066, - 0.018581482329561066, - 0.01392028150439728, - 0.01392028150439728, - 0.01392028150439728, - 0.01392028150439728, - 0.027841487986734137, - 0.027841487986734137, - 0.018460020675168682, - 0.018460020675168682, - 0.018460020675168682, - 0.01867730700178072, - 0.01867730700178072, - 0.01867730700178072, - 0.028588780987774953, - 0.028588780987774953, - 0.018283454002812505, - 0.018283454002812505, - 0.018283454002812505, - 0.02455506350088399, - 0.02455506350088399, - 0.02455506350088399, - 0.02455506350088399, - 0.018118704669177532, - 0.018118704669177532, - 0.018118704669177532, - 0.012909814744489267, - 0.012909814744489267, - 0.012909814744489267, - 0.012909814744489267, - 0.012955870508449152, - 0.012955870508449152, - 0.012955870508449152, - 0.012955870508449152, - 0.025617786479415372, - 0.025617786479415372, - 0.01710334767509873, - 0.01710334767509873, - 0.01710334767509873, - 0.01738610933534801, - 0.01738610933534801, - 0.01738610933534801, - 0.025520398980006576, - 0.025520398980006576, - 0.016854186678150047, - 0.016854186678150047, - 0.016854186678150047, - 0.01274280650250148, - 0.01274280650250148, - 0.01274280650250148, - 0.01274280650250148, - 0.02618332498241216, - 0.02618332498241216 - ], - [ - 0.07610312267206609, - 0.07610312267206609, - 0.07610312267206609, - 0.01501510600792244, - 0.01501510600792244, - 0.01501510600792244, - 0.01501510600792244, - 0.020073268989411492, - 0.020073268989411492, - 0.020073268989411492, - 0.020034901332110167, - 0.020034901332110167, - 0.020034901332110167, - 0.01501977325824555, - 0.01501977325824555, - 0.01501977325824555, - 0.01501977325824555, - 0.02973549149464816, - 0.02973549149464816, - 0.06096841796534136 - ] - ], - "generated_texts": [ - " the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed", - " dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit volupt", - " quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog.", - "piciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The", - " The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspic", - "atem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", - " jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", - "nis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis", - " error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over", - " voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde om" - ], - "errors": [ - "", - "", - "", - "", - "", - "", - "", - "", - "", - "" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-052659/report.md b/experiments/dsv4_p800_sglang/results/20260708-052659/report.md deleted file mode 100644 index a845a16..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-052659/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# Benchmark Report: dsv4_p800_sglang - -## Metadata - -- **Run ID**: 20260708-052659 -- **Timestamp**: 2026-07-08T05:26:59+00:00 -- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 -- **Engine/Backend**: sglang-xpu / sglang -- **Hardware**: 8x Kunlun P800 XPU -- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 -- **Git Commit**: 4ef6543 - -## Results - -| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-052659/results.json b/experiments/dsv4_p800_sglang/results/20260708-052659/results.json deleted file mode 100644 index ab11d28..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-052659/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-052659", - "timestamp": "2026-07-08T05:26:59+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "4ef6543", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-054332/results.json b/experiments/dsv4_p800_sglang/results/20260708-054332/results.json deleted file mode 100644 index 5c3f4eb..0000000 --- a/experiments/dsv4_p800_sglang/results/20260708-054332/results.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_p800_sglang", - "run_id": "20260708-054332", - "timestamp": "2026-07-08T05:43:32+00:00", - "model": "/data1/models/DeepSeek-V4-Flash-INT8", - "backend": "sglang", - "engine": "sglang-xpu", - "hardware": "8x Kunlun P800 XPU", - "accelerator": "Kunlun P800 XPU", - "chip": "kunlun_p800", - "script": "experiments/dsv4_p800_sglang/run_bench.sh", - "env": "", - "git_commit": "4ef6543", - "git_dirty": "dirty", - "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" - }, - "config": {}, - "scenarios": [] -} From b1b5f50f56106041632c54df41d657bdc9ebbaf0 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 04:08:40 +0000 Subject: [PATCH 08/53] results(p800): max context length exploration run 20260709-033134 P800 SGLang INT8 max supported input length: 131072 tokens. 262144 tokens fails during warmup (server connection lost, likely OOM). --- .../results/20260709-033134/report.md | 24 ++ .../results/20260709-033134/results.json | 315 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md create mode 100644 experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md b/experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md new file mode 100644 index 0000000..dfc6670 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md @@ -0,0 +1,24 @@ +# P800 Max Context Length Exploration Report + +- Result root: `/data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134` +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Hardware: 8x Kunlun P800 XPU +- TP: 8, EP: 8 +- Output len: 1, num prompts: 1, concurrency: 1 + +## Per-attempt results + +| Target len | Max context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4096 | 5120 | ✅ pass | 233.71 | 0.00 | 233.72 | 233.72 | | +| 8192 | 9216 | ✅ pass | 239.86 | 0.00 | 239.87 | 239.87 | | +| 16384 | 17408 | ✅ pass | 262.19 | 0.00 | 262.20 | 262.20 | | +| 32768 | 33792 | ✅ pass | 309.52 | 0.00 | 309.54 | 309.54 | | +| 65536 | 66560 | ✅ pass | 390.71 | 0.00 | 390.72 | 390.72 | | +| 131072 | 132096 | ✅ pass | 583.94 | 0.00 | 583.95 | 583.95 | | +| 262144 | 263168 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | bench_serving exited with code 137; see /data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134/logs/sglang_ctx_262144.log | + +## Maximum supported input length + +- **sglang-xpu**: 131072 tokens + diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json b/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json new file mode 100644 index 0000000..41ea383 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json @@ -0,0 +1,315 @@ +{ + "metadata": { + "experiment": "dsv4_p800_max_context_length", + "run_id": "20260709-033134", + "timestamp": "2026-07-09T03:31:34+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_max_context_length/run_bench.sh", + "env": "", + "git_commit": "407bc21", + "git_dirty": "dirty", + "description": "P800 max context length exploration for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024 + }, + "scenarios": [ + { + "backend": "sglang", + "target_len": 4096, + "max_context_len": 5120, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 5120 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 5120", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.3655072329565883, + "request_throughput": 0.2971320311564159, + "input_token_throughput": 1217.0527996166795, + "output_token_throughput": 0.2971320311564159, + "total_input_tokens": 4096, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 233.7192000122741, + "p50": 233.7192000122741, + "p90": 233.7192000122741, + "p95": 0.0, + "p99": 233.7192000122741 + }, + "ttft_ms": { + "mean": 233.70750999310985, + "p50": 233.70750999310985, + "p90": 0.0, + "p95": 0.0, + "p99": 233.70750999310985 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 8192, + "max_context_len": 9216, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 9216 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 9216", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.376425461960025, + "request_throughput": 0.2961712056926312, + "input_token_throughput": 2426.234517034035, + "output_token_throughput": 0.2961712056926312, + "total_input_tokens": 8192, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 239.87414099974558, + "p50": 239.87414099974558, + "p90": 239.87414099974558, + "p95": 0.0, + "p99": 239.87414099974558 + }, + "ttft_ms": { + "mean": 239.86148100811988, + "p50": 239.86148100811988, + "p90": 0.0, + "p95": 0.0, + "p99": 239.86148100811988 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 16384, + "max_context_len": 17408, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 17408 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 17408", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.4154617289896123, + "request_throughput": 0.292786182176261, + "input_token_throughput": 4797.00880877586, + "output_token_throughput": 0.292786182176261, + "total_input_tokens": 16384, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 262.1981970150955, + "p50": 262.1981970150955, + "p90": 262.1981970150955, + "p95": 0.0, + "p99": 262.1981970150955 + }, + "ttft_ms": { + "mean": 262.1856770128943, + "p50": 262.1856770128943, + "p90": 0.0, + "p95": 0.0, + "p99": 262.1856770128943 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 32768, + "max_context_len": 33792, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 33792 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 33792", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.431973898957949, + "request_throughput": 0.2913775073591411, + "input_token_throughput": 9547.858161144335, + "output_token_throughput": 0.2913775073591411, + "total_input_tokens": 32768, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 309.53772098291665, + "p50": 309.53772098291665, + "p90": 309.53772098291665, + "p95": 0.0, + "p99": 309.53772098291665 + }, + "ttft_ms": { + "mean": 309.5244810101576, + "p50": 309.5244810101576, + "p90": 0.0, + "p95": 0.0, + "p99": 309.5244810101576 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 65536, + "max_context_len": 66560, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 66560 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 66560", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.5316250980249606, + "request_throughput": 0.28315576320919333, + "input_token_throughput": 18556.896097677694, + "output_token_throughput": 0.28315576320919333, + "total_input_tokens": 65536, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 390.7171559985727, + "p50": 390.7171559985727, + "p90": 390.7171559985727, + "p95": 0.0, + "p99": 390.7171559985727 + }, + "ttft_ms": { + "mean": 390.70626598550007, + "p50": 390.70626598550007, + "p90": 0.0, + "p95": 0.0, + "p99": 390.70626598550007 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 131072, + "max_context_len": 132096, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 132096 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 132096", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.6164236619952135, + "request_throughput": 1.6222608923921629, + "input_token_throughput": 212632.97968762557, + "output_token_throughput": 1.6222608923921629, + "total_input_tokens": 131072, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 583.9470439823344, + "p50": 583.9470439823344, + "p90": 583.9470439823344, + "p95": 0.0, + "p99": 583.9470439823344 + }, + "ttft_ms": { + "mean": 583.9353539631702, + "p50": 583.9353539631702, + "p90": 0.0, + "p95": 0.0, + "p99": 583.9353539631702 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 262144, + "max_context_len": 263168, + "success": false, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 263168 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 263168", + "metrics": null, + "error": "bench_serving exited with code 137; see /data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134/logs/sglang_ctx_262144.log" + } + ] +} \ No newline at end of file From 90eecc4ede7dd40407b7814f941d99061fd74586 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 04:45:04 +0000 Subject: [PATCH 09/53] Add P800 long-context matrix experiment (64k/128k) for DeepSeek-V4-Flash-INT8 - New experiment: experiments/dsv4_p800_long_context_matrix/ - Single backend: sglang-xpu in Docker on Kunlun P800 - Context groups: 64k (max_context_len=80000, max_running=10) and 128k (max_context_len=140000, max_running=5) - Scenarios: concurrency sweeps x output lengths 256/1024/4096 - Parameterized start_sglang.sh enforces max_running >= 2 to avoid P800 INT8 req_to_token_pool allocation failures - Reuses proven P800 INT8 environment, NSA/EAGLE settings, and bench_serving invocations from dsv4_p800_max_context_length --- .../dsv4_p800_long_context_matrix/README.md | 33 +++ .../dsv4_p800_long_context_matrix/config.env | 51 ++++ .../run_bench.sh | 275 ++++++++++++++++++ .../start_sglang.sh | 133 +++++++++ 4 files changed, 492 insertions(+) create mode 100644 experiments/dsv4_p800_long_context_matrix/README.md create mode 100644 experiments/dsv4_p800_long_context_matrix/config.env create mode 100755 experiments/dsv4_p800_long_context_matrix/run_bench.sh create mode 100755 experiments/dsv4_p800_long_context_matrix/start_sglang.sh diff --git a/experiments/dsv4_p800_long_context_matrix/README.md b/experiments/dsv4_p800_long_context_matrix/README.md new file mode 100644 index 0000000..b8a77ce --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/README.md @@ -0,0 +1,33 @@ +# P800 Long-Context Matrix for DeepSeek-V4-Flash-INT8 + +Long-context throughput/latency matrix for SGLang-XPU on Kunlun P800 (8x XPU). + +## Scope + +- Model: `DeepSeek-V4-Flash-INT8` (W8A8 INT8) +- Backend/engine: `sglang-xpu` +- Hardware: 8x Kunlun P800 XPU +- Input lengths: 64k, 128k +- Output lengths: 256, 1024, 4096 +- Concurrency: sweeps per context group (see `config.env`) + +## Usage + +```bash +cd /data1/yy/sskj/experiments/dsv4_p800_long_context_matrix +bash run_bench.sh +``` + +Optional environment overrides: + +```bash +PORT=30013 RUN_ID=myrun bash run_bench.sh +``` + +## Outputs + +Results are saved under `results/${RUN_ID}/`: + +- `raw_outputs/`: raw `sglang.bench_serving` JSONL files +- `report.md`: parsed per-scenario metrics +- `results.json`: structured metadata + scenarios diff --git a/experiments/dsv4_p800_long_context_matrix/config.env b/experiments/dsv4_p800_long_context_matrix/config.env new file mode 100644 index 0000000..9d88604 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/config.env @@ -0,0 +1,51 @@ +# Long-context matrix for SGLang-XPU on DeepSeek-V4-Flash-INT8 (P800, TP=8). +# Input lengths: 64k, 128k (P800 INT8 max input length is ~131072 tokens). +# Output lengths: 256, 1k, 4k +# Concurrency: conservative to fit XPU memory. + +EXPERIMENT="dsv4_p800_long_context_matrix" +MODEL_NAME="DeepSeek-V4-Flash-INT8" +MODEL_PATH="/data1/models/DeepSeek-V4-Flash-INT8" +SERVED_MODEL_NAME="deepseek-v4-flash-int8" + +PORT="${PORT:-30013}" + +# Padding added to the longest sequence length when setting server context length. +CONTEXT_PAD="${CONTEXT_PAD:-1024}" + +# Per-context server settings. Each group is run with its own server start. +# Format: "input_label max_context_len max_running" +declare -a CONTEXT_GROUPS=( + "64k 80000 10" + "128k 140000 5" +) + +# Scenarios per group: "concurrency input_len output_len num_prompts" +declare -a SCENARIOS_64K=( + "10 65536 256 50" + "5 65536 256 25" + "2 65536 256 10" + "10 65536 1024 50" + "5 65536 1024 25" + "2 65536 1024 10" + "10 65536 4096 50" + "5 65536 4096 25" + "2 65536 4096 10" +) + +declare -a SCENARIOS_128K=( + "5 131072 256 20" + "2 131072 256 10" + "1 131072 256 5" + "5 131072 1024 20" + "2 131072 1024 10" + "1 131072 1024 5" + "5 131072 4096 20" + "2 131072 4096 10" + "1 131072 4096 5" +) + +CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}" +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" +PATCH_ROOT="${PATCH_ROOT:-${ROOT_DIR}/platforms/patches/kunlun_p800}" diff --git a/experiments/dsv4_p800_long_context_matrix/run_bench.sh b/experiments/dsv4_p800_long_context_matrix/run_bench.sh new file mode 100755 index 0000000..61ddb86 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/run_bench.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# Long-context matrix for P800 SGLang INT8. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RUN_DATE="$(date '+%m%d')" +RESULT_BASE="${SCRIPT_DIR}/results" +RESULT_ROOT="${RESULT_BASE}/${RUN_ID}" +LOG_DIR="${RESULT_ROOT}/logs" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "groups=${#CONTEXT_GROUPS[@]}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + log "stopping container ${CONTAINER_NAME}" + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + pkill -9 -f 'sglang.launch_server' 2>/dev/null || true + sleep 2 +} + +start_server() { + local max_context_len="$1" + local max_running="$2" + log "starting server (context-length=${max_context_len}, max-running=${max_running})" + bash "${SCRIPT_DIR}/start_sglang.sh" "$max_context_len" "$max_running" >> "${LOG_DIR}/start_server_${max_context_len}.log" 2>&1 +} + +run_warmup() { + local input_len="$1" + local output_len="${2:-256}" + local container_output="/tmp/bench_outputs/sglang_warmup_${input_len}_${output_len}.jsonl" + local detail_log="${LOG_DIR}/warmup_${input_len}_${output_len}.log" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + docker exec "$CONTAINER_NAME" mkdir -p "$(dirname "$container_output")" + + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "${CONTAINER_PYTHON}" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$PORT" \ + --model "$MODEL_PATH" \ + --dataset-name random \ + --dataset-path /workspace/dummy_sharegpt.json \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --random-range-ratio 1.0 \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file "$container_output" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "WARNING: warmup failed; see ${detail_log}" + return 1 + } + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$(python3 -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +run_group() { + local group_label="$1" + local max_context_len="$2" + local max_running="$3" + local -n scenarios_ref="$4" + + mkdir -p "$RAW_DIR" "$LOG_DIR" + + log "===== ${group_label} START (context-length=${max_context_len}, max-running=${max_running}) =====" + + stop_server + if ! start_server "$max_context_len" "$max_running"; then + log "ERROR: failed to start server for ${group_label}; skipping group" + stop_server + return 1 + fi + + # Warmup with the shortest output for this input length to save time. + local warmup_input="${scenarios_ref[0]}" + local warmup_input_len + warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')" + run_warmup "$warmup_input_len" 256 + + for scenario in "${scenarios_ref[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + + local host_output_file="${RAW_DIR}/sglang_${group_label}_${RUN_DATE}_${concurrency}_${input_len}_${output_len}.jsonl" + local container_output="/tmp/bench_outputs/sglang_${group_label}_${RUN_DATE}_${concurrency}_${input_len}_${output_len}.jsonl" + local detail_log="${LOG_DIR}/sglang_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log" + + if scenario_already_completed "$host_output_file" "$num_prompts"; then + log "skipping already-completed scenario: group=${group_label} c=${concurrency} i=${input_len} o=${output_len}" + continue + fi + + log "running scenario: group=${group_label} c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + + docker exec "$CONTAINER_NAME" mkdir -p "$(dirname "$container_output")" + + local bench_rc=0 + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "${CONTAINER_PYTHON}" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$PORT" \ + --model "$MODEL_PATH" \ + --dataset-name random \ + --dataset-path /workspace/dummy_sharegpt.json \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --random-range-ratio 1.0 \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$container_output" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + if [[ "$bench_rc" -ne 0 ]]; then + log "ERROR: scenario group=${group_label} c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + fi + + docker cp "${CONTAINER_NAME}:${container_output}" "$host_output_file" || { + log "ERROR: failed to copy output from container for scenario group=${group_label} c=${concurrency} i=${input_len} o=${output_len}" + continue + } + + # Verify the request actually completed. + local completed + completed="$(python3 -c " +import json +with open('${host_output_file}') as f: + for line in f: + data = json.loads(line) + print(data.get('completed', 0)) + break +")" + if [[ "${completed:-0}" -lt "$num_prompts" ]]; then + log "ERROR: scenario group=${group_label} c=${concurrency} i=${input_len} o=${output_len} only completed ${completed}/${num_prompts}" + continue + fi + + log "finished scenario: output=${host_output_file}" + done + + stop_server + log "===== ${group_label} DONE =====" +} + +parse_backend() { + log "parsing results in ${RESULT_ROOT}" + python3 "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$RESULT_ROOT" --backend sglang \ + >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parse_backend.py failed; see ${LOG_DIR}/parse.log" + } +} + +write_metadata() { + ensure_result_root "$RESULT_ROOT" + local meta_json="${RESULT_ROOT}/results.json" + + write_metadata_json \ + "$meta_json" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang-xpu" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "P800 long-context matrix for DeepSeek-V4-Flash-INT8" + + # Embed config. + python3 - "$meta_json" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "platform": "kunlun_p800", + "container_name": "sglang-dsv4-flash", + "context_pad": 1024, + "groups": [ + {"label": "64k", "max_context_len": 80000, "max_running": 10}, + {"label": "128k", "max_context_len": 140000, "max_running": 5} + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +stop_server +write_metadata + +for group in "${CONTEXT_GROUPS[@]}"; do + read -r group_label max_context_len max_running <<< "$group" + + # Resolve scenario array for this group. + scenario_array_name="SCENARIOS_${group_label^^}" + if [[ -z "${!scenario_array_name:-}" ]]; then + log "WARNING: no scenarios defined for group ${group_label}" + continue + fi + + run_group "$group_label" "$max_context_len" "$max_running" "$scenario_array_name" +done + +parse_backend + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh new file mode 100755 index 0000000..ac58709 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Start P800 SGLang INT8 server for the long-context matrix. +# Usage: start_sglang.sh +set -Eeuo pipefail + +MAX_CONTEXT_LEN="${1:-}" +MAX_RUNNING="${2:-}" +if [[ -z "$MAX_CONTEXT_LEN" || -z "$MAX_RUNNING" ]]; then + echo "Usage: $0 " + exit 1 +fi + +if [[ "$MAX_RUNNING" -lt 2 ]]; then + echo "ERROR: max_running must be >= 2 on P800 INT8 (1 causes req_to_token_pool allocation failures)" + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RESULT_ROOT="${RESULT_ROOT:-/tmp/${EXPERIMENT}}" +SERVER_LOG="${RESULT_ROOT}/logs/server_${MAX_CONTEXT_LEN}.outer.log" +mkdir -p "$(dirname "$SERVER_LOG")" + +log "starting P800 SGLang INT8 server (context-length=${MAX_CONTEXT_LEN}, max-running=${MAX_RUNNING})" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "server log: ${SERVER_LOG}" + +# Stop any existing container with the same name. +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build device args. +device_args="" +for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" +done +device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + +# Environment variables required by the P800 SGLang INT8 image. +env_args=( + -e XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + -e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + -e CUDA_DEVICE_ORDER=OAM_ID + -e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 + -e XMLIR_FORCE_USE_XPU_GRAPH=1 + -e SGLANG_DSV4_MODE=2604 + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + -e SGLANG_NSA_DUAL_STREAM=true + -e SGLANG_NSA_QUANT_WQ_B_WK=false + -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + -e SGLANG_OPT_USE_TILELANG_MHC_PRE=1 + -e SGLANG_OPT_USE_TILELANG_MHC_POST=1 + -e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 + -e SGLANG_SET_CPU_AFFINITY=1 + -e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 + -e XSGL_INTERTYPE_BFP16=1 + -e ENABLE_FAST_BFP16_ATTN=1 + -e XSGL_USE_DEEP_GEMM_BMM=1 + -e XSGL_XDNN_QUANT=1 + -e XSGL_FUSE_RMS_NORM_QUANT=1 + -e XSGL_TRANSPOSE_MATMUL_WEIGHT=1 + -e XINFER_QUANT_SDNN=1 + -e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 + -e XSGL_EARLY_FIRST_TOKEN=1 + -e XSGL_ENABLE_TGEMM_FP16=1 + -e SGLANG_ENABLE_SPEC_V2=True + -e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + -e PYTHONDONTWRITEBYTECODE=1 + -e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops + -e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib + -e BKCL_TREE_THRESHOLD=1048576 + -e CUDA_ENABLE_P2P_NO_UVA=1 + -e NCCL_IB_GID_INDEX=3 + -e IS_DSV4=1 + -e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json + # INT8 specific + -e SGLANG_DSV4_FP4_EXPERTS=false + -e SGLANG_APPLY_CONFIG_BACKUP=auto + -e BKCL_ENABLE_XDR=1 + -e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 + -e BKCL_RDMA_VERBS=1 + -e XSGL_INT8_LM_HEAD=1 + -e SGLANG_P800_ALL_GATHER_FALLBACK=0 +) + +# Launch args. Keep the proven P800 INT8 command from dsv4_p800_max_context_length, +# but make --context-length and --max-running-requests dynamic per group. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" + +# Base64-encode the bootstrap command to avoid host-shell quoting issues. +server_cmd=$(cat </dev/null || true +/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q +/root/miniconda/envs/python310_torch25_cuda/bin/pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q +/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" ${launch_args} +EOF +) +server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0) + +patch_mounts=( + -v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro" + -v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro" +) + +docker run -d \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --network host \ + --ipc host \ + ${device_args} \ + -v "${MODEL_PATH}:/models:ro" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + "${patch_mounts[@]}" \ + "${env_args[@]}" \ + "${DOCKER_IMAGE}" \ + bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \ + >> "${SERVER_LOG}" 2>&1 + +log "container ${CONTAINER_NAME} started, waiting for health" +if health_check 127.0.0.1 "$PORT" 600; then + log "container ${CONTAINER_NAME} is healthy" +else + log "ERROR: container ${CONTAINER_NAME} failed health check" + exit 1 +fi From 59f480b1bcb08123244e8a111de0d93ed90ac945 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 04:47:59 +0000 Subject: [PATCH 10/53] Disable chunked prefill and speculative decoding in P800 long-context matrix --- experiments/dsv4_p800_long_context_matrix/start_sglang.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh index ac58709..5812801 100755 --- a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh +++ b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh @@ -92,7 +92,8 @@ env_args=( # Launch args. Keep the proven P800 INT8 command from dsv4_p800_max_context_length, # but make --context-length and --max-running-requests dynamic per group. -launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" +# Note: disable chunked prefill and speculative decoding for the long-context matrix. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" # Base64-encode the bootstrap command to avoid host-shell quoting issues. server_cmd=$(cat < Date: Thu, 9 Jul 2026 05:13:04 +0000 Subject: [PATCH 11/53] Re-enable chunked prefill for P800 long-context matrix stability Without chunked prefill the server OOM/crashes (exit 137) during or immediately after concurrent 64k-context scenarios. Speculative decoding remains disabled as requested. --- experiments/dsv4_p800_long_context_matrix/start_sglang.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh index 5812801..4489ace 100755 --- a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh +++ b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh @@ -92,8 +92,9 @@ env_args=( # Launch args. Keep the proven P800 INT8 command from dsv4_p800_max_context_length, # but make --context-length and --max-running-requests dynamic per group. -# Note: disable chunked prefill and speculative decoding for the long-context matrix. -launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" +# Note: chunked prefill is required for stable concurrent long-context prefill on P800; +# speculative decoding is disabled as requested. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" # Base64-encode the bootstrap command to avoid host-shell quoting issues. server_cmd=$(cat < Date: Thu, 9 Jul 2026 05:59:28 +0000 Subject: [PATCH 12/53] Scale P800 long-context matrix to concurrency=1 for stability - P800 INT8 cannot sustain >1 concurrent long-context request without OOM; set all scenarios to concurrency=1. - Keep max_running=2 because max_running=1 triggers req_to_token_pool allocation failures in this image. - Reduce 64k max_context_len to 72000 (65536 + 4096 + pad). - Make run_bench.sh resilient to container death: detect dead container and skip remaining scenarios in the group instead of crashing the orchestrator. - Update README to document the concurrency limitation. --- .../dsv4_p800_long_context_matrix/README.md | 2 +- .../dsv4_p800_long_context_matrix/config.env | 32 +++++++------------ .../run_bench.sh | 21 ++++++++++-- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/experiments/dsv4_p800_long_context_matrix/README.md b/experiments/dsv4_p800_long_context_matrix/README.md index b8a77ce..c59e2e6 100644 --- a/experiments/dsv4_p800_long_context_matrix/README.md +++ b/experiments/dsv4_p800_long_context_matrix/README.md @@ -9,7 +9,7 @@ Long-context throughput/latency matrix for SGLang-XPU on Kunlun P800 (8x XPU). - Hardware: 8x Kunlun P800 XPU - Input lengths: 64k, 128k - Output lengths: 256, 1024, 4096 -- Concurrency: sweeps per context group (see `config.env`) +- Concurrency: **1** — P800 INT8 can only sustain one concurrent long-context request without OOM; `max_running=2` is kept because `max_running=1` triggers `req_to_token_pool` allocation failures in this image. ## Usage diff --git a/experiments/dsv4_p800_long_context_matrix/config.env b/experiments/dsv4_p800_long_context_matrix/config.env index 9d88604..0bde182 100644 --- a/experiments/dsv4_p800_long_context_matrix/config.env +++ b/experiments/dsv4_p800_long_context_matrix/config.env @@ -1,7 +1,9 @@ # Long-context matrix for SGLang-XPU on DeepSeek-V4-Flash-INT8 (P800, TP=8). # Input lengths: 64k, 128k (P800 INT8 max input length is ~131072 tokens). # Output lengths: 256, 1k, 4k -# Concurrency: conservative to fit XPU memory. +# Concurrency: P800 INT8 can only sustain 1 concurrent long-context request +# without OOM, so all scenarios use concurrency=1. max_running is kept at 2 +# because max_running=1 triggers req_to_token_pool allocation failures. EXPERIMENT="dsv4_p800_long_context_matrix" MODEL_NAME="DeepSeek-V4-Flash-INT8" @@ -16,33 +18,21 @@ CONTEXT_PAD="${CONTEXT_PAD:-1024}" # Per-context server settings. Each group is run with its own server start. # Format: "input_label max_context_len max_running" declare -a CONTEXT_GROUPS=( - "64k 80000 10" - "128k 140000 5" + "64k 72000 2" + "128k 140000 2" ) # Scenarios per group: "concurrency input_len output_len num_prompts" declare -a SCENARIOS_64K=( - "10 65536 256 50" - "5 65536 256 25" - "2 65536 256 10" - "10 65536 1024 50" - "5 65536 1024 25" - "2 65536 1024 10" - "10 65536 4096 50" - "5 65536 4096 25" - "2 65536 4096 10" + "1 65536 256 10" + "1 65536 1024 10" + "1 65536 4096 10" ) declare -a SCENARIOS_128K=( - "5 131072 256 20" - "2 131072 256 10" - "1 131072 256 5" - "5 131072 1024 20" - "2 131072 1024 10" - "1 131072 1024 5" - "5 131072 4096 20" - "2 131072 4096 10" - "1 131072 4096 5" + "1 131072 256 5" + "1 131072 1024 5" + "1 131072 4096 5" ) CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}" diff --git a/experiments/dsv4_p800_long_context_matrix/run_bench.sh b/experiments/dsv4_p800_long_context_matrix/run_bench.sh index 61ddb86..4b4c7d2 100755 --- a/experiments/dsv4_p800_long_context_matrix/run_bench.sh +++ b/experiments/dsv4_p800_long_context_matrix/run_bench.sh @@ -38,6 +38,10 @@ is_server_healthy() { curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 } +container_running() { + docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 +} + stop_server() { log "stopping container ${CONTAINER_NAME}" docker rm -f "$CONTAINER_NAME" 2>/dev/null || true @@ -59,6 +63,10 @@ run_warmup() { local detail_log="${LOG_DIR}/warmup_${input_len}_${output_len}.log" log "warming up (input=${input_len}, output=${output_len}, num=1)" + if ! container_running; then + log "WARNING: container not running, skipping warmup" + return 1 + fi docker exec "$CONTAINER_NAME" mkdir -p "$(dirname "$container_output")" docker exec "$CONTAINER_NAME" \ @@ -142,6 +150,11 @@ run_group() { continue fi + if ! container_running; then + log "WARNING: container died, skipping remaining scenarios in group=${group_label}" + break + fi + log "running scenario: group=${group_label} c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" docker exec "$CONTAINER_NAME" mkdir -p "$(dirname "$container_output")" @@ -168,6 +181,10 @@ run_group() { if [[ "$bench_rc" -ne 0 ]]; then log "ERROR: scenario group=${group_label} c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + if ! container_running; then + log "WARNING: container died, skipping remaining scenarios in group=${group_label}" + break + fi continue fi @@ -241,8 +258,8 @@ data["config"] = { "container_name": "sglang-dsv4-flash", "context_pad": 1024, "groups": [ - {"label": "64k", "max_context_len": 80000, "max_running": 10}, - {"label": "128k", "max_context_len": 140000, "max_running": 5} + {"label": "64k", "max_context_len": 72000, "max_running": 2}, + {"label": "128k", "max_context_len": 140000, "max_running": 2} ] } with open(path, "w", encoding="utf-8") as f: From 1482601ae5c77cc1d54f6e4c7731dd44b3a1d738 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 09:26:24 +0000 Subject: [PATCH 13/53] =?UTF-8?q?Add=20num=5Fprompts=20=3D=205=C3=97concur?= =?UTF-8?q?rency=20convention=20and=20update=20all=20experiments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Document the rule in docs/EXPERIMENT_GUIDE.md section 7 - Update all config.env SCENARIOS to follow the 5x rule - Migrate dsv4_h200_sglang, dsv4_h200_vllm, dsv4_p800_sglang from global NUM_PROMPTS to per-scenario num_prompts - Keep long-context phase2 as exception (low counts documented) - Add legacy 3-field fallback in run_bench.sh loops --- docs/EXPERIMENT_GUIDE.md | 21 ++++++++- experiments/TEMPLATE/config.env | 5 ++- .../dsv4_h200_256k_4k_probe/config.env | 3 +- .../dsv4_h200_64k_sglang_vs_vllm/config.env | 5 ++- .../dsv4_h200_long_context_matrix/config.env | 44 ++++++++++--------- experiments/dsv4_h200_sglang/config.env | 10 +++-- experiments/dsv4_h200_sglang/run_bench.sh | 13 +++--- .../dsv4_h200_sglang_vs_vllm/config.env | 16 ++++--- experiments/dsv4_h200_vllm/config.env | 14 +++--- experiments/dsv4_h200_vllm/run_bench.sh | 13 +++--- .../config.env | 11 ++--- .../dsv4_h200_vllm_mtp_vs_default/config.env | 13 +++--- experiments/dsv4_p800_sglang/config.env | 10 ++--- experiments/dsv4_p800_sglang/run_bench.sh | 12 +++-- 14 files changed, 116 insertions(+), 74 deletions(-) diff --git a/docs/EXPERIMENT_GUIDE.md b/docs/EXPERIMENT_GUIDE.md index 9609f5e..a772634 100644 --- a/docs/EXPERIMENT_GUIDE.md +++ b/docs/EXPERIMENT_GUIDE.md @@ -155,14 +155,31 @@ git commit -m "results: run " git push origin main ``` -## 7. 命名约定 +## 7. 场景设计规范 + +### 7.1 请求数 = 并发数 × 5 + +每个 scenario 的 `num_prompts` 必须是对应 `concurrency` 的 **5 倍**，即每个并发档位至少跑满 5 轮请求，保证 P95/P99 等尾延迟指标有统计意义。 + +```bash +# 格式：concurrency input_len output_len num_prompts +declare -a SCENARIOS=( + "1 512 256 5" # 1 × 5 + "32 512 256 160" # 32 × 5 + "128 512 256 640" # 128 × 5 +) +``` + +例外：对于超长上下文（如 128k/256k）或探针类实验，若执行成本过高，可在 `config.env` 中显式注释说明原因，并保留不小于 `concurrency × 2` 的最低样本量。 + +## 8. 命名约定 - 实验目录：`{model}_{chip}_{scenario}_{backend_vs_backend}`，例如 `dsv4_h200_sglang_vs_vllm` - scenario 名称：`c{concurrency}_i{input_len}_o{output_len}` - raw 输出：`{backend}_{label}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl` - run_id：`YYYYMMDD-HHMMSS` -## 8. 常见问题 +## 9. 常见问题 ### 8.1 新芯片上 server 起不来 diff --git a/experiments/TEMPLATE/config.env b/experiments/TEMPLATE/config.env index 8d6df3c..f75bcad 100644 --- a/experiments/TEMPLATE/config.env +++ b/experiments/TEMPLATE/config.env @@ -22,9 +22,10 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" MAX_RUNNING="${MAX_RUNNING:-256}" # Scenarios: "concurrency input_len output_len num_prompts". +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. declare -a SCENARIOS=( - "1 512 256 32" - "32 512 256 128" + "1 512 256 5" + "32 512 256 160" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_256k_4k_probe/config.env b/experiments/dsv4_h200_256k_4k_probe/config.env index b9a72e6..2994862 100644 --- a/experiments/dsv4_h200_256k_4k_probe/config.env +++ b/experiments/dsv4_h200_256k_4k_probe/config.env @@ -22,8 +22,9 @@ MAX_NUM_SEQS=8 MAX_RUNNING=8 # "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. declare -a SCENARIOS=( - "2 262144 4096 4" + "2 262144 4096 10" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env b/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env index c64aff4..d4caad7 100644 --- a/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env +++ b/experiments/dsv4_h200_64k_sglang_vs_vllm/config.env @@ -23,9 +23,10 @@ MAX_NUM_SEQS=64 MAX_RUNNING=64 # "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. declare -a SCENARIOS=( - "25 65536 256 100" - "8 65536 1024 100" + "25 65536 256 125" + "8 65536 1024 40" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_long_context_matrix/config.env b/experiments/dsv4_h200_long_context_matrix/config.env index fcddcf5..37f9325 100644 --- a/experiments/dsv4_h200_long_context_matrix/config.env +++ b/experiments/dsv4_h200_long_context_matrix/config.env @@ -32,34 +32,36 @@ declare -a CONTEXT_GROUPS=( ) # Scenarios per group: "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. +# NOTE: long-context scenarios are expensive; these counts follow the 5x rule. declare -a SCENARIOS_64K=( - "25 65536 256 50" - "25 65536 1024 50" - "25 65536 4096 50" + "25 65536 256 125" + "25 65536 1024 125" + "25 65536 4096 125" ) declare -a SCENARIOS_128K=( - "10 131072 256 20" - "5 131072 256 10" - "2 131072 256 4" - "10 131072 1024 20" - "5 131072 1024 10" - "2 131072 1024 4" - "10 131072 4096 20" - "5 131072 4096 10" - "2 131072 4096 4" + "10 131072 256 50" + "5 131072 256 25" + "2 131072 256 10" + "10 131072 1024 50" + "5 131072 1024 25" + "2 131072 1024 10" + "10 131072 4096 50" + "5 131072 4096 25" + "2 131072 4096 10" ) declare -a SCENARIOS_256K=( - "10 262144 256 20" - "5 262144 256 10" - "2 262144 256 4" - "10 262144 1024 20" - "5 262144 1024 10" - "2 262144 1024 4" - "10 262144 4096 20" - "5 262144 4096 10" - "2 262144 4096 4" + "10 262144 256 50" + "5 262144 256 25" + "2 262144 256 10" + "10 262144 1024 50" + "5 262144 1024 25" + "2 262144 1024 10" + "10 262144 4096 50" + "5 262144 4096 25" + "2 262144 4096 10" ) VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_sglang/config.env b/experiments/dsv4_h200_sglang/config.env index 2fc51bb..60e0f43 100644 --- a/experiments/dsv4_h200_sglang/config.env +++ b/experiments/dsv4_h200_sglang/config.env @@ -15,14 +15,16 @@ ENGINE="sglang" VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/sglang}" VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# Benchmark scenarios: each element is "concurrency input_len output_len". +# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts". +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. if [[ -z "${SCENARIOS:-}" ]]; then SCENARIOS=( - "32 512 256" - "128 512 256" + "32 512 256 160" + "128 512 256 640" ) fi -NUM_PROMPTS="${NUM_PROMPTS:-128}" +# Legacy fallback: used only if SCENARIOS entries have 3 fields. +NUM_PROMPTS="${NUM_PROMPTS:-160}" # Server start script bundled with this experiment. SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_sglang/run_bench.sh b/experiments/dsv4_h200_sglang/run_bench.sh index 18b4044..8c494b1 100755 --- a/experiments/dsv4_h200_sglang/run_bench.sh +++ b/experiments/dsv4_h200_sglang/run_bench.sh @@ -60,8 +60,7 @@ data["config"] = { "tp": 8, "moe_runner_backend": "marlin", "port": 30006, - "num_prompts": 128, - "scenarios": ["32 512 256", "128 512 256"] + "scenarios": ["32 512 256 160", "128 512 256 640"] } with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) @@ -126,11 +125,15 @@ fi log "===== BENCHMARK START =====" for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len <<< "$scenario" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi output_file="${RAW_DIR}/sglang_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log" - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ --backend sglang \ @@ -139,7 +142,7 @@ for scenario in "${SCENARIOS[@]}"; do --dataset-name random \ --random-input-len "$input_len" \ --random-output-len "$output_len" \ - --num-prompts "$NUM_PROMPTS" \ + --num-prompts "$num_prompts" \ --max-concurrency "$concurrency" \ --request-rate 10000 \ --output-file "$output_file" \ diff --git a/experiments/dsv4_h200_sglang_vs_vllm/config.env b/experiments/dsv4_h200_sglang_vs_vllm/config.env index 7d1708f..d7c7c07 100644 --- a/experiments/dsv4_h200_sglang_vs_vllm/config.env +++ b/experiments/dsv4_h200_sglang_vs_vllm/config.env @@ -20,13 +20,14 @@ TP=8 # Phase 1: short-context throughput (max_model_len 32k). # Each element: "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. if [[ -z "${PHASE1_SCENARIOS:-}" ]]; then declare -a PHASE1_SCENARIOS=( - "1 512 256 32" - "32 512 256 128" - "128 512 256 128" - "1 4000 512 32" - "32 4000 512 64" + "1 512 256 5" + "32 512 256 160" + "128 512 256 640" + "1 4000 512 5" + "32 4000 512 160" ) fi PHASE1_MAX_MODEL_LEN=32768 @@ -34,6 +35,7 @@ PHASE1_MAX_NUM_SEQS=256 PHASE1_MAX_RUNNING=256 # Phase 2: long context (max_model_len 210k to leave room for output + padding). +# Exception to the 5x rule due to very long input lengths; kept at low counts. if [[ -z "${PHASE2_SCENARIOS:-}" ]]; then declare -a PHASE2_SCENARIOS=( "1 32768 1024 8" @@ -51,8 +53,8 @@ SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_ SHAREGPT_CONTEXT_LEN=32768 if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then declare -a SHAREGPT_SCENARIOS=( - "1 32768 256 32" - "32 32768 256 128" + "1 32768 256 5" + "32 32768 256 160" ) fi diff --git a/experiments/dsv4_h200_vllm/config.env b/experiments/dsv4_h200_vllm/config.env index cda9047..46f1073 100644 --- a/experiments/dsv4_h200_vllm/config.env +++ b/experiments/dsv4_h200_vllm/config.env @@ -15,17 +15,19 @@ ENGINE="vllm" VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/vllm}" VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# Benchmark scenarios: each element is "concurrency input_len output_len". +# Benchmark scenarios: each element is "concurrency input_len output_len num_prompts". +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. # Override via SCENARIOS env var as a bash array, e.g.: -# SCENARIOS=("32 512 256" "128 512 256") +# SCENARIOS=("32 512 256 160" "128 512 256 640") if [[ -z "${SCENARIOS:-}" ]]; then SCENARIOS=( - "32 512 256" - "128 512 256" - "32 4000 512" + "32 512 256 160" + "128 512 256 640" + "32 4000 512 160" ) fi -NUM_PROMPTS="${NUM_PROMPTS:-128}" +# Legacy fallback: used only if SCENARIOS entries have 3 fields. +NUM_PROMPTS="${NUM_PROMPTS:-160}" # Server start script bundled with this experiment. SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm/run_bench.sh b/experiments/dsv4_h200_vllm/run_bench.sh index 12f5245..257f57b 100755 --- a/experiments/dsv4_h200_vllm/run_bench.sh +++ b/experiments/dsv4_h200_vllm/run_bench.sh @@ -62,8 +62,7 @@ data["config"] = { "block_size": 256, "max_num_seqs": 256, "port": 30005, - "num_prompts": 128, - "scenarios": ["32 512 256", "128 512 256", "32 4000 512"] + "scenarios": ["32 512 256 160", "128 512 256 640", "32 4000 512 160"] } with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) @@ -128,11 +127,15 @@ fi log "===== BENCHMARK START =====" for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len <<< "$scenario" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ --backend vllm \ @@ -141,7 +144,7 @@ for scenario in "${SCENARIOS[@]}"; do --dataset-name random \ --random-input-len "$input_len" \ --random-output-len "$output_len" \ - --num-prompts "$NUM_PROMPTS" \ + --num-prompts "$num_prompts" \ --max-concurrency "$concurrency" \ --request-rate 10000 \ --output-file "$output_file" \ diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/config.env b/experiments/dsv4_h200_vllm_dspark_vs_default/config.env index ce532b5..e900986 100644 --- a/experiments/dsv4_h200_vllm_dspark_vs_default/config.env +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/config.env @@ -30,13 +30,14 @@ TP=8 # Scenarios focused on TTFT: low/high concurrency x short/medium input. # Each element: "concurrency input_len output_len num_prompts" +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. if [[ -z "${SCENARIOS:-}" ]]; then declare -a SCENARIOS=( - "1 512 256 32" - "32 512 256 128" - "128 512 256 128" - "1 4000 512 32" - "32 4000 512 64" + "1 512 256 5" + "32 512 256 160" + "128 512 256 640" + "1 4000 512 5" + "32 4000 512 160" ) fi MAX_MODEL_LEN=32768 diff --git a/experiments/dsv4_h200_vllm_mtp_vs_default/config.env b/experiments/dsv4_h200_vllm_mtp_vs_default/config.env index a6ee34f..34c73e8 100644 --- a/experiments/dsv4_h200_vllm_mtp_vs_default/config.env +++ b/experiments/dsv4_h200_vllm_mtp_vs_default/config.env @@ -21,12 +21,15 @@ SPEC_METHOD="${SPEC_METHOD:-mtp}" SPEC_TOKENS="${SPEC_TOKENS:-1}" # Scenarios must match the ones in the reused default result set. +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. +# NOTE: if these scenarios are changed, the default-side results must be re-run +# with the same scenarios (update DEFAULT_RESULT_ROOT accordingly). declare -a SCENARIOS=( - "1 512 256 32" - "32 512 256 128" - "128 512 256 128" - "1 4000 512 32" - "32 4000 512 64" + "1 512 256 5" + "32 512 256 160" + "128 512 256 640" + "1 4000 512 5" + "32 4000 512 160" ) MTP_START_SCRIPT="${SCRIPT_DIR:-.}/start_mtp.sh" diff --git a/experiments/dsv4_p800_sglang/config.env b/experiments/dsv4_p800_sglang/config.env index f984eb8..919bcaa 100644 --- a/experiments/dsv4_p800_sglang/config.env +++ b/experiments/dsv4_p800_sglang/config.env @@ -24,13 +24,13 @@ NUM_PROMPTS="${NUM_PROMPTS:-10}" # - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint SERVER_MODE="${SERVER_MODE:-w8a8_int8}" -# Scenario list: "concurrency input_len output_len" (one per line). -# Default to a single smoke-test scenario for fast validation. Override to run -# a full grid, e.g.: -# SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256") +# Scenario list: "concurrency input_len output_len num_prompts" (one per line). +# Following docs/EXPERIMENT_GUIDE.md: num_prompts = concurrency * 5. +# Default to a single scenario for fast validation. Override to run a full grid, e.g.: +# SCENARIOS=("32 512 256 160" "128 512 256 640") if [[ -z "${SCENARIOS:-}" ]]; then SCENARIOS=( - "32 512 256" + "32 512 256 160" ) fi diff --git a/experiments/dsv4_p800_sglang/run_bench.sh b/experiments/dsv4_p800_sglang/run_bench.sh index 3b8d29e..bbd3abe 100755 --- a/experiments/dsv4_p800_sglang/run_bench.sh +++ b/experiments/dsv4_p800_sglang/run_bench.sh @@ -72,15 +72,19 @@ trap on_exit EXIT log "===== BENCHMARK START =====" for scenario in "${SCENARIOS[@]}"; do - # Allow SCENARIOS to be passed as a string like "(32 512 256)" from env. + # Allow SCENARIOS to be passed as a string like "(32 512 256 160)" from env. scenario="${scenario//[()]/}" - read -r concurrency input_len output_len <<< "$scenario" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" host_output_file="${RAW_DIR}/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${LOG_DIR}/${BACKEND}_c${concurrency}_i${input_len}_o${output_len}.log" - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" run_random_case \ "$BACKEND" \ @@ -90,7 +94,7 @@ for scenario in "${SCENARIOS[@]}"; do "$concurrency" \ "$input_len" \ "$output_len" \ - "$NUM_PROMPTS" \ + "$num_prompts" \ "${DATASET_PATH:-}" \ > "$detail_log" 2>&1 || { log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed" From 04a384b265419349a5ae9f8db4620246ac200ebe Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 09:55:18 +0000 Subject: [PATCH 14/53] =?UTF-8?q?add=20dsv4=5Fh200=5Fvllm=5Ftp=5Fdp=5Fmatr?= =?UTF-8?q?ix=20experiment=20(TP=C3=97DP=20matrix=20with=20Y-only=20mode,?= =?UTF-8?q?=201M=20excluded)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dsv4_h200_vllm_tp_dp_matrix/compare.py | 162 +++++++ .../dsv4_h200_vllm_tp_dp_matrix/config.env | 43 ++ .../generate_scenarios.py | 105 +++++ .../dsv4_h200_vllm_tp_dp_matrix/matrix.json | 81 ++++ .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 394 ++++++++++++++++++ .../smoke_matrix.json | 18 + .../start_vllm_dp.sh | 92 ++++ scripts/common/parse_backend.py | 97 ++++- 8 files changed, 987 insertions(+), 5 deletions(-) create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/config.env create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py new file mode 100755 index 0000000..8bf26df --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h200_vllm_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "✅" + if ttft_ok or tpot_ok: + return "⚠️" + return "❌" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: vLLM\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, x.replace("c", "").replace("i", "_").replace("o", "_").split("_")[1:]))): + cfg_part, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + # cfg_part not used; just for readability. + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/DSL. + f.write("\n## Best throughput per (ISL, DSL)\n\n") + f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + isl_i, dsl_i = int(isl), int(dsl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]: + best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s) + for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env new file mode 100644 index 0000000..b9f0398 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -0,0 +1,43 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs). +# Tests vLLM with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +VLLM_PORT="${VLLM_PORT:-30030}" +# vLLM 0.24.0 multi-port DP supervisor listens on this hard-coded port. +VLLM_DP_SUPERVISOR_PORT="${VLLM_DP_SUPERVISOR_PORT:-9256}" + +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" +VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" +# The benchmark client is sglang.bench_serving, even when the backend is vLLM. +VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# Matrix and concurrency rules are defined in matrix.json. +# This file is consumed by generate_scenarios.py. +MATRIX_FILE="${SCRIPT_DIR:-.}/matrix.json" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. 0 means use the default heuristic in +# generate_scenarios.py (6-8 points, or all integers when range is small). +CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py new file mode 100755 index 0000000..ef46e01 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return a sorted list of concurrency values in [low, high]. + + If the range is small, return every integer. Otherwise sample roughly + `target` points linearly between low and high (inclusive). + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if high - low + 1 <= target: + return list(range(low, high + 1)) + + points = set() + points.add(low) + points.add(high) + step = (high - low) / (target - 1) + for i in range(1, target - 1): + v = low + round(step * i) + points.add(max(low, min(high, v))) + return sorted(points) + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + dsl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for dsl_str in sorted(dsl_map.keys(), key=int): + mark = dsl_map[dsl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(dsl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..1ed8e9f --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json @@ -0,0 +1,81 @@ +{ + "comment": "ISL/DSL matrix for dsv4_h200_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "N", + "256": "N", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 }, + "262144": { "low": 1, "high": 2 }, + "524288": { "low": 1, "high": 2 }, + "1048576": { "low": 1, "high": 2 } + } +} diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..0df25b4 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -0,0 +1,394 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y+P}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +PYTHON="${VENV_CLIENT}/bin/python" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +health_port_for() { + local dp="$1" + if [[ "$dp" -gt 1 ]]; then + echo "$VLLM_DP_SUPERVISOR_PORT" + else + echo "$VLLM_PORT" + fi +} + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + + # Fallback: kill any vllm serve processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + local max_model_len="$3" + local max_num_seqs="$4" + + local args=( + "vllm serve" "$MODEL_PATH" + --trust-remote-code + --tensor-parallel-size "$tp" + --kv-cache-dtype fp8 + --max-model-len "$max_model_len" + --max-num-seqs "$max_num_seqs" + --block-size 256 + --gpu-memory-utilization 0.90 + --tokenizer-mode deepseek_v4 + --reasoning-parser deepseek_v4 + --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --port "$VLLM_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --data-parallel-size "$dp" + --data-parallel-size-local "$dp" + --data-parallel-multi-port-external-lb + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + local max_model_len="$3" + local max_num_seqs="$4" + + log "starting vllm server tp=${tp} dp=${dp} max_model_len=${max_model_len} max_num_seqs=${max_num_seqs}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" "$max_model_len" "$max_num_seqs" \ + >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + local hport + hport="$(health_port_for "$dp")" + if ! is_server_healthy "$hport"; then + log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${hport}" + return 1 + fi + log "vllm server tp=${tp} dp=${dp} is healthy on port ${hport}" +} + +run_warmup() { + local tp="$1" + local dp="$2" + local input_len="$3" + local output_len="$4" + local hport + hport="$(health_port_for "$dp")" + + log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$hport" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num 1 \ + --env-python "$PYTHON" \ + >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.warmup.log" 2>&1 + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + # Remaining args are key=value pairs. + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {k: json.loads(v) for k, v in pairs} +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + if [[ "$total_scenarios" -eq 0 ]]; then + log "no scenarios for ${config_label}; skipping" + return 0 + fi + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "vllm" \ + "vllm" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_VLLM" \ + "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" + + # Embed static config now; server_args will be updated per ISL group. + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "vllm", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh" + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + # Group scenarios by input_len. We start one server per input_len with a + # max_model_len large enough for the biggest output_len in that group. + local current_isl="" + local max_dsl_for_isl=0 + local max_conc_for_isl=0 + local group_started=false + + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + # When the input length changes, restart the server for the new group. + if [[ "$isl" != "$current_isl" ]]; then + if [[ "$group_started" == true ]]; then + stop_server "$tp" "$dp" + fi + current_isl="$isl" + max_dsl_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($3>max) max=$3} END{print max+0}' "$scenario_tsv")" + max_conc_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($4>max) max=$4} END{print max+0}' "$scenario_tsv")" + + local max_model_len=$((isl + max_dsl_for_isl + 64)) + local max_num_seqs=$((max_conc_for_isl + 8)) + + stop_server "$tp" "$dp" + if ! start_server "$tp" "$dp" "$max_model_len" "$max_num_seqs"; then + log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group" + group_started=false + continue + fi + group_started=true + + # Update metadata with the actual server args for this ISL group. + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp" "$max_model_len" "$max_num_seqs")" + jq --arg isl "$isl" --arg args "$server_args_str" \ + '.config.server_args_per_isl += {($isl): $args}' \ + "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + # Warmup with the shortest output for this ISL. + run_warmup "$tp" "$dp" "$isl" 128 + fi + + # If the server for this group did not start, skip all scenarios in it. + if [[ "$group_started" != true ]]; then + log "skipping ${config_label} ISL=${isl} scenario (server not started)" + continue + fi + + local output_file="${raw_dir}/vllm_main_c${conc}_i${isl}_o${dsl}.jsonl" + local detail_log="${phase_log_dir}/vllm_${config_label}_c${conc}_i${isl}_o${dsl}.log" + local gpu_csv="${gpu_log_dir}/gpu_mem_c${conc}_i${isl}_o${dsl}.csv" + + if scenario_already_completed "$output_file" "$num"; then + log "skipping already-completed ${config_label} scenario: c=${conc} i=${isl} o=${dsl}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + local bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" \ + "$PYTHON" -m sglang.bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$(health_port_for "$dp")" \ + --dataset-name random \ + --random-input-len "$isl" \ + --random-output-len "$dsl" \ + --num-prompts "$num" \ + --max-concurrency "$conc" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -ne 0 ]]; then + log "ERROR: ${config_label} scenario c=${conc} i=${isl} o=${dsl} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped_oom and continuing" + append_scenario_record "$result_root" \ + "name=c${conc}_i${isl}_o${dsl}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_oom\"" \ + "note=\"bench command failed or timed out (rc=${bench_rc})\"" + continue + else + log "mandatory (Y) scenario failed; aborting current ISL group" + stop_server "$tp" "$dp" + group_started=false + continue + fi + fi + + log "finished ${config_label} scenario: output=${output_file}" + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json new file mode 100644 index 0000000..5715c28 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json @@ -0,0 +1,18 @@ +{ + "comment": "Small smoke matrix for quickly verifying TP×DP server startup and bench path.", + "mode": "Y", + "matrix": { + "1024": { + "128": "Y", + "256": "Y" + }, + "4096": { + "128": "Y", + "256": "Y" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 8 }, + "4096": { "low": 1, "high": 4 } + } +} diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh new file mode 100755 index 0000000..2ae2fee --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Start vLLM server for a given TP×DP configuration. +# Usage: start_vllm_dp.sh +set -e + +TP="${1}" +DP="${2}" +MAX_MODEL_LEN="${3}" +MAX_NUM_SEQS="${4}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +# Build the base command as an array. +SERVER_ARGS=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --tensor-parallel-size "$TP" + --kv-cache-dtype fp8 + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --block-size 256 + --gpu-memory-utilization 0.90 + --tokenizer-mode deepseek_v4 + --reasoning-parser deepseek_v4 + --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --port "$VLLM_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + --data-parallel-size-local "$DP" + --data-parallel-multi-port-external-lb + ) + # vLLM 0.24.0 hard-codes the DP supervisor port to 9256 in + # entrypoints/openai/cli_args.py. The bench client and health checks must + # target that port when DP > 1. + HEALTH_PORT="$VLLM_DP_SUPERVISOR_PORT" +else + HEALTH_PORT="$VLLM_PORT" +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}, max_model_len=${MAX_MODEL_LEN}, max_num_seqs=${MAX_NUM_SEQS}) ===" +echo "Model: $MODEL_PATH" +echo "Health port: $HEALTH_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${HEALTH_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${HEALTH_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${HEALTH_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/scripts/common/parse_backend.py b/scripts/common/parse_backend.py index f07e6c6..d269dc1 100755 --- a/scripts/common/parse_backend.py +++ b/scripts/common/parse_backend.py @@ -93,6 +93,72 @@ def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: floa } +def parse_gpu_memory_csv(jsonl_path: Path) -> dict | None: + """Parse a paired nvidia-smi CSV for GPU memory/utilization statistics. + + The CSV is expected to live in a sibling `gpu_logs/` directory or in the same + `raw_outputs/` directory, named `gpu_mem_c_i_o.csv`. + """ + result_root = jsonl_path.parent.parent + scenario_id = jsonl_path.stem.split("_", 2)[2] # e.g. c1_i1024_o128 + csv_name = f"gpu_mem_{scenario_id}.csv" + + csv_path = None + for candidate in ( + result_root / "gpu_logs" / csv_name, + result_root / "raw_outputs" / csv_name, + ): + if candidate.exists() and candidate.stat().st_size > 0: + csv_path = candidate + break + + if csv_path is None: + return None + + per_gpu = {} + total_mb = None + try: + with open(csv_path, "r", encoding="utf-8") as f: + header = f.readline() + if not header.strip(): + return None + for line in f: + line = line.strip() + if not line: + continue + parts = [p.strip() for p in line.split(",")] + if len(parts) < 5: + continue + idx = parts[1] + try: + used = float(parts[2].split()[0]) + total = float(parts[3].split()[0]) + util = float(parts[4].split()[0]) + except (ValueError, IndexError): + continue + per_gpu.setdefault(idx, {"used": [], "util": []}) + per_gpu[idx]["used"].append(used) + per_gpu[idx]["util"].append(util) + if total_mb is None: + total_mb = total + except Exception: + return None + + if not per_gpu or total_mb is None: + return None + + peak_used = max(max(g["used"]) for g in per_gpu.values()) + avg_used = sum(sum(g["used"]) / len(g["used"]) for g in per_gpu.values()) / len(per_gpu) + peak_util = max(max(g["util"]) for g in per_gpu.values()) + + return { + "peak_used_mb": peak_used, + "avg_used_mb": avg_used, + "peak_utilization_pct": peak_util, + "memory_total_mb": total_mb, + } + + def generate_report(result_root: Path, backend: str, scenarios: list[dict], metadata: dict | None) -> None: report_path = result_root / "report.md" model = metadata.get("model", "unknown") if metadata else "unknown" @@ -106,13 +172,18 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") f.write("## Results\n\n") - f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") for s in scenarios: cfg = s["config"] m = s["metrics"] slo = s.get("slo_status", {}).get("overall", "") + gpu = m.get("gpu_memory") + if gpu: + gpu_str = f"{gpu['peak_used_mb']:.0f}/{gpu['memory_total_mb']:.0f} MiB ({100*gpu['peak_used_mb']/gpu['memory_total_mb']:.1f}%)" + else: + gpu_str = "-" f.write( f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " @@ -120,7 +191,7 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta f"{m['total_token_throughput']:.2f} | " f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {gpu_str} | {slo} |\n" ) f.write("\n") f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") @@ -152,10 +223,13 @@ def main() -> None: raise SystemExit("Could not infer backend from raw outputs") metadata = None + old_scenarios = [] if results_json.exists(): with open(results_json, "r", encoding="utf-8") as f: try: - metadata = json.load(f).get("metadata") + existing = json.load(f) + metadata = existing.get("metadata") + old_scenarios = existing.get("scenarios", []) except json.JSONDecodeError: pass @@ -183,6 +257,7 @@ def main() -> None: continue metrics = compute_metrics(data) + metrics["gpu_memory"] = parse_gpu_memory_csv(jsonl_path) scenario = { "name": scenario_name(concurrency, input_len, output_len), "config": { @@ -199,10 +274,22 @@ def main() -> None: } scenarios.append(scenario) - if not scenarios: + if not scenarios and not old_scenarios: print("No benchmark outputs found to parse") return + # Preserve any manually-recorded skipped/failed scenarios (e.g. optional + # combinations that OOMed) that do not have a fresh raw output. + parsed_names = {s["name"] for s in scenarios} + for old in old_scenarios: + if old.get("status") and old["name"] not in parsed_names: + scenarios.append(old) + scenarios.sort(key=lambda s: ( + s["config"]["input_len"], + s["config"]["output_len"], + s["config"]["concurrency"], + )) + if results_json.exists(): with open(results_json, "r", encoding="utf-8") as f: data = json.load(f) From 15b548953590f2408e82e15bee6559fbc1f51990 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:01:51 +0000 Subject: [PATCH 15/53] dsv4_h200_vllm_tp_dp_matrix: use minimal official vLLM args, no EP, Y-only, exclude 1M --- .../tp2_dp4/results.json | 29 +++ .../tp2_dp4/scenarios.tsv | 166 ++++++++++++++++++ .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 50 ++---- .../start_vllm_dp.sh | 29 ++- 4 files changed, 222 insertions(+), 52 deletions(-) create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json new file mode 100644 index 0000000..d101ad8 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json @@ -0,0 +1,29 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp_dp_matrix_tp2_dp4", + "run_id": "20260709-095536_smoke", + "timestamp": "2026-07-09T09:55:43+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "ec85ef1", + "git_dirty": "dirty", + "description": "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 2, + "dp": 4, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args_per_isl": { + "1024": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 2 --kv-cache-dtype fp8 --max-model-len 5184 --max-num-seqs 136 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30030 --data-parallel-size 4 --data-parallel-size-local 4 --data-parallel-multi-port-external-lb " + } + }, + "scenarios": [] +} diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv new file mode 100644 index 0000000..a064e0a --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv @@ -0,0 +1,166 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 22 110 +Y 1024 128 43 215 +Y 1024 128 65 325 +Y 1024 128 86 430 +Y 1024 128 107 535 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 22 110 +Y 1024 256 43 215 +Y 1024 256 65 325 +Y 1024 256 86 430 +Y 1024 256 107 535 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 22 110 +Y 1024 512 43 215 +Y 1024 512 65 325 +Y 1024 512 86 430 +Y 1024 512 107 535 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 22 110 +Y 1024 1024 43 215 +Y 1024 1024 65 325 +Y 1024 1024 86 430 +Y 1024 1024 107 535 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 22 110 +Y 1024 2048 43 215 +Y 1024 2048 65 325 +Y 1024 2048 86 430 +Y 1024 2048 107 535 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 22 110 +Y 1024 4096 43 215 +Y 1024 4096 65 325 +Y 1024 4096 86 430 +Y 1024 4096 107 535 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 11 55 +Y 4096 128 22 110 +Y 4096 128 33 165 +Y 4096 128 43 215 +Y 4096 128 53 265 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 11 55 +Y 4096 256 22 110 +Y 4096 256 33 165 +Y 4096 256 43 215 +Y 4096 256 53 265 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 11 55 +Y 4096 512 22 110 +Y 4096 512 33 165 +Y 4096 512 43 215 +Y 4096 512 53 265 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 11 55 +Y 4096 1024 22 110 +Y 4096 1024 33 165 +Y 4096 1024 43 215 +Y 4096 1024 53 265 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 11 55 +Y 4096 2048 22 110 +Y 4096 2048 33 165 +Y 4096 2048 43 215 +Y 4096 2048 53 265 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 11 55 +Y 4096 4096 22 110 +Y 4096 4096 33 165 +Y 4096 4096 43 215 +Y 4096 4096 53 265 +Y 4096 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 6 30 +Y 16384 128 11 55 +Y 16384 128 17 85 +Y 16384 128 22 110 +Y 16384 128 27 135 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 6 30 +Y 16384 256 11 55 +Y 16384 256 17 85 +Y 16384 256 22 110 +Y 16384 256 27 135 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 6 30 +Y 16384 512 11 55 +Y 16384 512 17 85 +Y 16384 512 22 110 +Y 16384 512 27 135 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 6 30 +Y 16384 1024 11 55 +Y 16384 1024 17 85 +Y 16384 1024 22 110 +Y 16384 1024 27 135 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 6 30 +Y 16384 2048 11 55 +Y 16384 2048 17 85 +Y 16384 2048 22 110 +Y 16384 2048 27 135 +Y 16384 2048 32 160 +Y 65536 128 1 5 +Y 65536 128 2 10 +Y 65536 128 3 15 +Y 65536 128 5 25 +Y 65536 128 6 30 +Y 65536 128 7 35 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 2 10 +Y 65536 256 3 15 +Y 65536 256 5 25 +Y 65536 256 6 30 +Y 65536 256 7 35 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 2 10 +Y 65536 512 3 15 +Y 65536 512 5 25 +Y 65536 512 6 30 +Y 65536 512 7 35 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 2 10 +Y 65536 1024 3 15 +Y 65536 1024 5 25 +Y 65536 1024 6 30 +Y 65536 1024 7 35 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 2 10 +Y 131072 128 3 15 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 2 10 +Y 131072 256 3 15 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 2 10 +Y 131072 512 3 15 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index 0df25b4..6e3ed6e 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -15,7 +15,7 @@ source "${SCRIPT_DIR}/config.env" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" -MATRIX_MODE="${MATRIX_MODE:-Y+P}" +MATRIX_MODE="${MATRIX_MODE:-Y}" SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" @@ -72,22 +72,14 @@ stop_server() { build_server_args() { local tp="$1" local dp="$2" - local max_model_len="$3" - local max_num_seqs="$4" local args=( "vllm serve" "$MODEL_PATH" --trust-remote-code - --tensor-parallel-size "$tp" --kv-cache-dtype fp8 - --max-model-len "$max_model_len" - --max-num-seqs "$max_num_seqs" --block-size 256 - --gpu-memory-utilization 0.90 - --tokenizer-mode deepseek_v4 - --reasoning-parser deepseek_v4 - --no-disable-hybrid-kv-cache-manager - --disable-uvicorn-access-log + --tensor-parallel-size "$tp" + --no-enable-flashinfer-autotune --port "$VLLM_PORT" ) if [[ "$dp" -gt 1 ]]; then @@ -103,11 +95,9 @@ build_server_args() { start_server() { local tp="$1" local dp="$2" - local max_model_len="$3" - local max_num_seqs="$4" - log "starting vllm server tp=${tp} dp=${dp} max_model_len=${max_model_len} max_num_seqs=${max_num_seqs}" - bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" "$max_model_len" "$max_num_seqs" \ + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" \ >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 local hport @@ -244,22 +234,23 @@ run_parallel_config() { "$VENV_VLLM" \ "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" - # Embed static config now; server_args will be updated per ISL group. - jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" \ + # Embed static config and the exact server command line used. + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ '.config = { "tp": ($tp | tonumber), "dp": ($dp | tonumber), "cuda_visible_devices": $cuda, "backend": "vllm", - "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh" + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh", + "server_args": $args }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ mv "${result_root}/results.json.tmp" "${result_root}/results.json" - # Group scenarios by input_len. We start one server per input_len with a - # max_model_len large enough for the biggest output_len in that group. + # Group scenarios by input_len. We restart the server for each ISL to keep + # memory state isolated and to warm up per context length. local current_isl="" - local max_dsl_for_isl=0 - local max_conc_for_isl=0 local group_started=false tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do @@ -269,28 +260,15 @@ run_parallel_config() { stop_server "$tp" "$dp" fi current_isl="$isl" - max_dsl_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($3>max) max=$3} END{print max+0}' "$scenario_tsv")" - max_conc_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($4>max) max=$4} END{print max+0}' "$scenario_tsv")" - - local max_model_len=$((isl + max_dsl_for_isl + 64)) - local max_num_seqs=$((max_conc_for_isl + 8)) stop_server "$tp" "$dp" - if ! start_server "$tp" "$dp" "$max_model_len" "$max_num_seqs"; then + if ! start_server "$tp" "$dp"; then log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group" group_started=false continue fi group_started=true - # Update metadata with the actual server args for this ISL group. - local server_args_str - server_args_str="$(build_server_args "$tp" "$dp" "$max_model_len" "$max_num_seqs")" - jq --arg isl "$isl" --arg args "$server_args_str" \ - '.config.server_args_per_isl += {($isl): $args}' \ - "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ - mv "${result_root}/results.json.tmp" "${result_root}/results.json" - # Warmup with the shortest output for this ISL. run_warmup "$tp" "$dp" "$isl" 128 fi diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh index 2ae2fee..a4e52e0 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -1,12 +1,18 @@ #!/usr/bin/env bash # Start vLLM server for a given TP×DP configuration. -# Usage: start_vllm_dp.sh +# Usage: start_vllm_dp.sh +# +# Uses the minimal argument set from the official DeepSeek-V4-Flash recipe: +# --trust-remote-code +# --kv-cache-dtype fp8 +# --block-size 256 +# --tensor-parallel-size +# --no-enable-flashinfer-autotune +# plus DP-related flags when DP > 1. set -e TP="${1}" DP="${2}" -MAX_MODEL_LEN="${3}" -MAX_NUM_SEQS="${4}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null @@ -26,20 +32,13 @@ PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" -# Build the base command as an array. SERVER_ARGS=( vllm serve "$MODEL_PATH" --trust-remote-code - --tensor-parallel-size "$TP" --kv-cache-dtype fp8 - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --block-size 256 - --gpu-memory-utilization 0.90 - --tokenizer-mode deepseek_v4 - --reasoning-parser deepseek_v4 - --no-disable-hybrid-kv-cache-manager - --disable-uvicorn-access-log + --tensor-parallel-size "$TP" + --no-enable-flashinfer-autotune --port "$VLLM_PORT" ) @@ -49,9 +48,7 @@ if [[ "$DP" -gt 1 ]]; then --data-parallel-size-local "$DP" --data-parallel-multi-port-external-lb ) - # vLLM 0.24.0 hard-codes the DP supervisor port to 9256 in - # entrypoints/openai/cli_args.py. The bench client and health checks must - # target that port when DP > 1. + # vLLM 0.24.0 hard-codes the DP supervisor port to 9256. HEALTH_PORT="$VLLM_DP_SUPERVISOR_PORT" else HEALTH_PORT="$VLLM_PORT" @@ -59,7 +56,7 @@ fi SERVER_ARGS_STR="${SERVER_ARGS[*]}" -echo "=== Starting vLLM server (TP=${TP}, DP=${DP}, max_model_len=${MAX_MODEL_LEN}, max_num_seqs=${MAX_NUM_SEQS}) ===" +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ===" echo "Model: $MODEL_PATH" echo "Health port: $HEALTH_PORT" echo "Command: $SERVER_ARGS_STR" From 39326eaf4aec3198b46b92b06b273f1e74395ab0 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:02:03 +0000 Subject: [PATCH 16/53] remove partial smoke test artifacts from killed run --- .../tp2_dp4/results.json | 29 --- .../tp2_dp4/scenarios.tsv | 166 ------------------ 2 files changed, 195 deletions(-) delete mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json deleted file mode 100644 index d101ad8..0000000 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/results.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp_dp_matrix_tp2_dp4", - "run_id": "20260709-095536_smoke", - "timestamp": "2026-07-09T09:55:43+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "ec85ef1", - "git_dirty": "dirty", - "description": "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" - }, - "config": { - "tp": 2, - "dp": 4, - "cuda_visible_devices": "0,1,2,3,4,5,6,7", - "backend": "vllm", - "server_start_script": "experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh", - "server_args_per_isl": { - "1024": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 2 --kv-cache-dtype fp8 --max-model-len 5184 --max-num-seqs 136 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30030 --data-parallel-size 4 --data-parallel-size-local 4 --data-parallel-multi-port-external-lb " - } - }, - "scenarios": [] -} diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv deleted file mode 100644 index a064e0a..0000000 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-095536_smoke/tp2_dp4/scenarios.tsv +++ /dev/null @@ -1,166 +0,0 @@ -mark input_len output_len concurrency num_prompts -Y 1024 128 1 5 -Y 1024 128 22 110 -Y 1024 128 43 215 -Y 1024 128 65 325 -Y 1024 128 86 430 -Y 1024 128 107 535 -Y 1024 128 128 640 -Y 1024 256 1 5 -Y 1024 256 22 110 -Y 1024 256 43 215 -Y 1024 256 65 325 -Y 1024 256 86 430 -Y 1024 256 107 535 -Y 1024 256 128 640 -Y 1024 512 1 5 -Y 1024 512 22 110 -Y 1024 512 43 215 -Y 1024 512 65 325 -Y 1024 512 86 430 -Y 1024 512 107 535 -Y 1024 512 128 640 -Y 1024 1024 1 5 -Y 1024 1024 22 110 -Y 1024 1024 43 215 -Y 1024 1024 65 325 -Y 1024 1024 86 430 -Y 1024 1024 107 535 -Y 1024 1024 128 640 -Y 1024 2048 1 5 -Y 1024 2048 22 110 -Y 1024 2048 43 215 -Y 1024 2048 65 325 -Y 1024 2048 86 430 -Y 1024 2048 107 535 -Y 1024 2048 128 640 -Y 1024 4096 1 5 -Y 1024 4096 22 110 -Y 1024 4096 43 215 -Y 1024 4096 65 325 -Y 1024 4096 86 430 -Y 1024 4096 107 535 -Y 1024 4096 128 640 -Y 4096 128 1 5 -Y 4096 128 11 55 -Y 4096 128 22 110 -Y 4096 128 33 165 -Y 4096 128 43 215 -Y 4096 128 53 265 -Y 4096 128 64 320 -Y 4096 256 1 5 -Y 4096 256 11 55 -Y 4096 256 22 110 -Y 4096 256 33 165 -Y 4096 256 43 215 -Y 4096 256 53 265 -Y 4096 256 64 320 -Y 4096 512 1 5 -Y 4096 512 11 55 -Y 4096 512 22 110 -Y 4096 512 33 165 -Y 4096 512 43 215 -Y 4096 512 53 265 -Y 4096 512 64 320 -Y 4096 1024 1 5 -Y 4096 1024 11 55 -Y 4096 1024 22 110 -Y 4096 1024 33 165 -Y 4096 1024 43 215 -Y 4096 1024 53 265 -Y 4096 1024 64 320 -Y 4096 2048 1 5 -Y 4096 2048 11 55 -Y 4096 2048 22 110 -Y 4096 2048 33 165 -Y 4096 2048 43 215 -Y 4096 2048 53 265 -Y 4096 2048 64 320 -Y 4096 4096 1 5 -Y 4096 4096 11 55 -Y 4096 4096 22 110 -Y 4096 4096 33 165 -Y 4096 4096 43 215 -Y 4096 4096 53 265 -Y 4096 4096 64 320 -Y 16384 128 1 5 -Y 16384 128 6 30 -Y 16384 128 11 55 -Y 16384 128 17 85 -Y 16384 128 22 110 -Y 16384 128 27 135 -Y 16384 128 32 160 -Y 16384 256 1 5 -Y 16384 256 6 30 -Y 16384 256 11 55 -Y 16384 256 17 85 -Y 16384 256 22 110 -Y 16384 256 27 135 -Y 16384 256 32 160 -Y 16384 512 1 5 -Y 16384 512 6 30 -Y 16384 512 11 55 -Y 16384 512 17 85 -Y 16384 512 22 110 -Y 16384 512 27 135 -Y 16384 512 32 160 -Y 16384 1024 1 5 -Y 16384 1024 6 30 -Y 16384 1024 11 55 -Y 16384 1024 17 85 -Y 16384 1024 22 110 -Y 16384 1024 27 135 -Y 16384 1024 32 160 -Y 16384 2048 1 5 -Y 16384 2048 6 30 -Y 16384 2048 11 55 -Y 16384 2048 17 85 -Y 16384 2048 22 110 -Y 16384 2048 27 135 -Y 16384 2048 32 160 -Y 65536 128 1 5 -Y 65536 128 2 10 -Y 65536 128 3 15 -Y 65536 128 5 25 -Y 65536 128 6 30 -Y 65536 128 7 35 -Y 65536 128 8 40 -Y 65536 256 1 5 -Y 65536 256 2 10 -Y 65536 256 3 15 -Y 65536 256 5 25 -Y 65536 256 6 30 -Y 65536 256 7 35 -Y 65536 256 8 40 -Y 65536 512 1 5 -Y 65536 512 2 10 -Y 65536 512 3 15 -Y 65536 512 5 25 -Y 65536 512 6 30 -Y 65536 512 7 35 -Y 65536 512 8 40 -Y 65536 1024 1 5 -Y 65536 1024 2 10 -Y 65536 1024 3 15 -Y 65536 1024 5 25 -Y 65536 1024 6 30 -Y 65536 1024 7 35 -Y 65536 1024 8 40 -Y 131072 128 1 5 -Y 131072 128 2 10 -Y 131072 128 3 15 -Y 131072 128 4 20 -Y 131072 256 1 5 -Y 131072 256 2 10 -Y 131072 256 3 15 -Y 131072 256 4 20 -Y 131072 512 1 5 -Y 131072 512 2 10 -Y 131072 512 3 15 -Y 131072 512 4 20 -Y 262144 128 1 5 -Y 262144 128 2 10 -Y 262144 256 1 5 -Y 262144 256 2 10 -Y 524288 128 1 5 -Y 524288 128 2 10 From 5898d578ed1039307593a95a5c1f7dc5767095d8 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:02:45 +0000 Subject: [PATCH 17/53] dsv4_h200_vllm_tp_dp_matrix: allow MATRIX_FILE env override --- experiments/dsv4_h200_vllm_tp_dp_matrix/config.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index b9f0398..51401bc 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -27,9 +27,9 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) -# Matrix and concurrency rules are defined in matrix.json. +# Matrix and concurrency rules are defined in matrix.json by default. # This file is consumed by generate_scenarios.py. -MATRIX_FILE="${SCRIPT_DIR:-.}/matrix.json" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" # Sampling density for concurrency. 0 means use the default heuristic in From 59dbadb1106354b076c61fb4b6787a015fe2dcc2 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:19:30 +0000 Subject: [PATCH 18/53] add dp_proxy for vLLM DP, write kv_cache_analysis.md, fix DP bench port --- .../kv_cache_analysis.md | 100 ++++++++++ .../dsv4_h200_vllm_tp_dp_matrix/config.env | 3 + .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 109 ++++++++++- scripts/common/dp_proxy.py | 173 ++++++++++++++++++ 4 files changed, 378 insertions(+), 7 deletions(-) create mode 100644 experiments/dsv4_h200_max_context_length/kv_cache_analysis.md create mode 100644 scripts/common/dp_proxy.py diff --git a/experiments/dsv4_h200_max_context_length/kv_cache_analysis.md b/experiments/dsv4_h200_max_context_length/kv_cache_analysis.md new file mode 100644 index 0000000..8b31fcb --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/kv_cache_analysis.md @@ -0,0 +1,100 @@ +# DeepSeek-V4-Flash 长上下文显存分析 + +> 针对 `dsv4_h200_max_context_length` 实验结果（8×H200，TP=8，output_len=1，concurrency=1）的显存复盘。 + +## 1. 关键模型参数 + +| 参数 | 数值 | +|---|---| +| 总参数量 | ~158B（safetensors 汇总） | +| Checkpoint 大小 | ~149 GiB（fp8 权重 + scale） | +| 层数 `num_hidden_layers` | 43 | +| Hidden size | 4096 | +| Attention heads | 64 | +| KV heads | 1 | +| 每层 KV `head_dim` | 512 | +| KV cache dtype | fp8 | +| 每层压缩比 | 2 层不压缩，21 层 4:1，20 层 128:1 | +| `max_position_embeddings` | **1,048,576** | +| H200 单卡显存 | 143 GiB | + +## 2. 理论显存估算 + +### 2.1 权重 + +``` +149 GiB / 8 (TP) ≈ 18.6 GiB / GPU +``` + +vLLM 实际启动日志：`Model loading took 20.07 GiB memory`，与理论值基本吻合（包含少量加载开销）。 + +### 2.2 KV cache + +按压缩后的平均每层 KV 大小估算： + +- 不压缩的 2 层：512 bytes/token +- 4:1 压缩的 21 层：128 bytes/token +- 128:1 压缩的 20 层：4 bytes/token + +``` +平均每层 = (2×512 + 21×128 + 20×4) / 43 ≈ 88 bytes/token +43 层 ≈ 3.8 KiB/token +``` + +再考虑 indexer、alignment 等辅助缓存，vLLM 实际日志给出的数字是： + +``` +Available KV cache memory: 101.39 GiB +GPU KV cache size: 14,668,151 tokens +=> 约 7.0 KiB / token / GPU +``` + +因此 1M 上下文的 KV cache 约为： + +``` +1,048,576 × 7.0 KiB ≈ 7.0 GiB / GPU +``` + +### 2.3 1M 上下文单卡总占用 + +| 组件 | 估算 | +|---|---| +| 权重 | ~20 GiB | +| KV cache（1M tokens） | ~7 GiB | +| CUDA graph / 启动开销 | ~1 GiB | +| 激活 / 临时 buffer | 数 GiB（prefill 峰值） | +| **合计** | **~30 GiB / GPU** | + +vLLM 可用显存按 `--gpu-memory-utilization 0.90` 计算： + +``` +143 GiB × 0.9 ≈ 128.7 GiB +``` + +**显存完全够用。** + +## 3. 与实测结果对比 + +| 实测 target_len | max_model_len | 结果 | +|---|---|---| +| 917,504 | 918,528 | ✅ 通过 | +| 1,048,576 | 1,049,600 | ❌ server failed to start | + +vLLM 1M 失败日志的关键信息： + +```text +User-specified max_model_len (1049600) is greater than the derived max_model_len +(max_position_embeddings=1048576 or model_max_length=None in model's config.json). +``` + +这说明： + +1. **917,504 通过是合理的**：此时 KV cache 约 `6.1 GiB/GPU`，加上权重和开销远不到显存上限。 +2. **1,048,576 失败是合理的**：实验里 `max_model_len = input_len + 1024 pad = 1,049,600`，超过了模型 config 声明的 `max_position_embeddings=1,048,576`。**这是位置编码/配置上限，不是显存上限**。 +3. 实验报告的最大长度 **917,504 是偏保守的**：理论上在 output_len=1 时，最大 input length 可以接近 `1,048,575`（只要 `max_model_len ≤ 1,048,576`）。实验因为采用了固定步长并加了 1024 的 pad，没有测到这个边界。 + +## 4. 结论 + +- **从显存角度看，1M 上下文在 8×H200 上完全可以放下**：单卡约 30 GiB，仅占可用显存的 1/4 左右。 +- **真正限制到不了 1M 的是 `max_position_embeddings=1048576`**，而不是显存。 +- `dsv4_h200_max_context_length` 的结果与理论估算一致；测出的 917,504 是“该实验配置下能跑通的最大值”，但不是“硬件/显存能支持的最大值”。 diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 51401bc..a02e497 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -12,6 +12,9 @@ SERVED_MODEL_NAME="deepseek-v4-flash" VLLM_PORT="${VLLM_PORT:-30030}" # vLLM 0.24.0 multi-port DP supervisor listens on this hard-coded port. VLLM_DP_SUPERVISOR_PORT="${VLLM_DP_SUPERVISOR_PORT:-9256}" +# The bench client does NOT talk to the supervisor directly; we put a +# round-robin proxy in front of the DP child servers. +VLLM_DP_PROXY_PORT="${VLLM_DP_PROXY_PORT:-30040}" VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index 6e3ed6e..99aa94b 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -32,7 +32,8 @@ log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}" # Helpers # --------------------------------------------------------------------------- -health_port_for() { +# Port used by run_bench to check the vLLM server (supervisor for DP>1). +server_health_port() { local dp="$1" if [[ "$dp" -gt 1 ]]; then echo "$VLLM_DP_SUPERVISOR_PORT" @@ -41,16 +42,35 @@ health_port_for() { fi } +# Port used by the benchmark client (proxy for DP>1). +bench_port() { + local dp="$1" + if [[ "$dp" -gt 1 ]]; then + echo "$VLLM_DP_PROXY_PORT" + else + echo "$VLLM_PORT" + fi +} + is_server_healthy() { local port="$1" curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 } +is_proxy_healthy() { + local port="$1" + # The bench client uses /v1/models as the ready signal, so the proxy must + # forward it successfully. + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/v1/models" >/dev/null 2>&1 +} + stop_server() { local tp="$1" local dp="$2" - local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + stop_proxy "$tp" "$dp" + + local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" if [[ -f "$pid_file" ]]; then local pid pid="$(cat "$pid_file")" @@ -69,6 +89,74 @@ stop_server() { sleep 2 } +start_proxy() { + local tp="$1" + local dp="$2" + if [[ "$dp" -le 1 ]]; then + return 0 + fi + + local proxy_port="$VLLM_DP_PROXY_PORT" + local base_port="$VLLM_PORT" + local backends="" + for ((r = 0; r < dp; r++)); do + backends="${backends:+,}$((base_port + r))" + done + + local pid_file="/data/user1/yy/${EXPERIMENT}_proxy_tp${tp}_dp${dp}.pid" + rm -f "$pid_file" + + log "starting DP proxy on port ${proxy_port} -> backends ${backends}" + nohup "${VENV_VLLM}/bin/python" "${SCRIPT_DIR}/../../scripts/common/dp_proxy.py" \ + --port "$proxy_port" \ + --backends "$backends" \ + --timeout 300 \ + > "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" 2>&1 & + + local pid=$! + echo $pid > "$pid_file" + + for i in $(seq 1 60); do + if is_proxy_healthy "$proxy_port"; then + log "DP proxy is ready on port ${proxy_port}" + return 0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + log "ERROR: DP proxy exited early" + tail -100 "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" + return 1 + fi + sleep 1 + done + + log "ERROR: DP proxy not healthy after 60s" + tail -100 "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" + return 1 +} + +stop_proxy() { + local tp="$1" + local dp="$2" + if [[ "$dp" -le 1 ]]; then + return 0 + fi + + local pid_file="/data/user1/yy/${EXPERIMENT}_proxy_tp${tp}_dp${dp}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping DP proxy pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + pkill -9 -f "dp_proxy.py.*--port ${VLLM_DP_PROXY_PORT}" 2>/dev/null || true + sleep 1 +} + build_server_args() { local tp="$1" local dp="$2" @@ -101,12 +189,19 @@ start_server() { >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 local hport - hport="$(health_port_for "$dp")" + hport="$(server_health_port "$dp")" if ! is_server_healthy "$hport"; then log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${hport}" return 1 fi log "vllm server tp=${tp} dp=${dp} is healthy on port ${hport}" + + if [[ "$dp" -gt 1 ]]; then + if ! start_proxy "$tp" "$dp"; then + log "error: failed to start DP proxy for tp=${tp} dp=${dp}" + return 1 + fi + fi } run_warmup() { @@ -114,14 +209,14 @@ run_warmup() { local dp="$2" local input_len="$3" local output_len="$4" - local hport - hport="$(health_port_for "$dp")" + local bport + bport="$(bench_port "$dp")" log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)" "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend vllm \ --host 127.0.0.1 \ - --port "$hport" \ + --port "$bport" \ --input-len "$input_len" \ --output-len "$output_len" \ --num 1 \ @@ -298,7 +393,7 @@ run_parallel_config() { "$PYTHON" -m sglang.bench_serving \ --backend vllm \ --host 127.0.0.1 \ - --port "$(health_port_for "$dp")" \ + --port "$(bench_port "$dp")" \ --dataset-name random \ --random-input-len "$isl" \ --random-output-len "$dsl" \ diff --git a/scripts/common/dp_proxy.py b/scripts/common/dp_proxy.py new file mode 100644 index 0000000..ff69bb1 --- /dev/null +++ b/scripts/common/dp_proxy.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Lightweight round-robin reverse proxy for vLLM data-parallel child servers. + +vLLM's --data-parallel-multi-port-external-lb only exposes aggregated /health on +a supervisor port; it does not forward /v1/models, /v1/completions, etc. This +proxy sits in front of the child API servers and round-robins requests across +them. + +Usage: + python3 dp_proxy.py --port 30040 --backends 30030,30031,30032,30033 +""" +import argparse +import asyncio +import itertools +from contextlib import asynccontextmanager + +import httpx +from fastapi import FastAPI, Request, Response +from fastapi.responses import StreamingResponse +from uvicorn import Config, Server + + +HOP_BY_HOP_HEADERS = { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "content-encoding", + "content-length", +} + + +class DPProxy: + def __init__(self, backends: list[int], timeout: float = 60.0): + self.backend_urls = [f"http://127.0.0.1:{port}" for port in backends] + self.cycle = itertools.cycle(self.backend_urls) + self.lock = asyncio.Lock() + self.timeout = timeout + self.client = httpx.AsyncClient( + timeout=httpx.Timeout(timeout), + limits=httpx.Limits(max_connections=100, max_keepalive_connections=20), + ) + + async def close(self): + await self.client.aclose() + + async def _next_backend(self) -> str: + async with self.lock: + return next(self.cycle) + + async def _is_healthy(self, base_url: str) -> bool: + try: + r = await self.client.get(f"{base_url}/health", timeout=2.0) + return r.status_code == 200 + except Exception: + return False + + async def health(self) -> Response: + results = await asyncio.gather( + *(self._is_healthy(url) for url in self.backend_urls) + ) + if all(results): + return Response(status_code=200) + return Response(status_code=503) + + async def ready(self) -> Response: + # Bench clients use /v1/models as the ready signal. We forward that + # to a healthy backend; this endpoint just checks /health. + return await self.health() + + async def proxy(self, request: Request) -> Response: + path = request.url.path + if request.url.query: + path = f"{path}?{request.url.query}" + + body = await request.body() + headers = dict(request.headers) + headers.pop("host", None) + + tried = set() + while len(tried) < len(self.backend_urls): + base_url = await self._next_backend() + if base_url in tried: + # Avoid infinite loop when only one backend remains. + break + tried.add(base_url) + + target = f"{base_url}{path}" + try: + backend_req = self.client.build_request( + method=request.method, + url=target, + headers=headers, + content=body, + ) + backend_resp = await self.client.send( + backend_req, follow_redirects=False, stream=True + ) + except Exception: + continue + + if backend_resp.status_code >= 500: + await backend_resp.aclose() + continue + + response_headers = { + k: v + for k, v in backend_resp.headers.items() + if k.lower() not in HOP_BY_HOP_HEADERS + } + return StreamingResponse( + backend_resp.aiter_bytes(), + status_code=backend_resp.status_code, + headers=response_headers, + background=None, + ) + + return Response( + content=b"All DP backends are unavailable", + status_code=503, + ) + + +def make_app(proxy: DPProxy) -> FastAPI: + app = FastAPI() + + @app.get("/health", include_in_schema=False) + async def health(): + return await proxy.health() + + @app.get("/ready", include_in_schema=False) + @app.get("/readyz", include_in_schema=False) + async def ready(): + return await proxy.ready() + + @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"]) + async def catch_all(request: Request): + return await proxy.proxy(request) + + return app + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, required=True) + parser.add_argument( + "--backends", + type=lambda s: [int(x.strip()) for x in s.split(",")], + required=True, + help="Comma-separated list of child vLLM server ports", + ) + parser.add_argument("--timeout", type=float, default=300.0) + parser.add_argument("--log-level", default="warning") + args = parser.parse_args() + + proxy = DPProxy(args.backends, timeout=args.timeout) + app = make_app(proxy) + + config = Config(app, host="0.0.0.0", port=args.port, log_level=args.log_level) + server = Server(config) + + try: + await server.serve() + finally: + await proxy.close() + + +if __name__ == "__main__": + asyncio.run(main()) From 329d4c39750ee1c688a03d852b47e098dc8008e6 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:28:50 +0000 Subject: [PATCH 19/53] dsv4_h200_vllm_tp_dp_matrix: use vLLM internal DP LB, drop proxy/supervisor --- .../dsv4_h200_vllm_tp_dp_matrix/config.env | 5 - .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 123 +----------------- .../start_vllm_dp.sh | 17 +-- 3 files changed, 12 insertions(+), 133 deletions(-) diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index a02e497..60a147d 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -10,11 +10,6 @@ MODEL_PATH="/data/models/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" VLLM_PORT="${VLLM_PORT:-30030}" -# vLLM 0.24.0 multi-port DP supervisor listens on this hard-coded port. -VLLM_DP_SUPERVISOR_PORT="${VLLM_DP_SUPERVISOR_PORT:-9256}" -# The bench client does NOT talk to the supervisor directly; we put a -# round-robin proxy in front of the DP child servers. -VLLM_DP_PROXY_PORT="${VLLM_DP_PROXY_PORT:-30040}" VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index 99aa94b..e3cdaa1 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -32,44 +32,14 @@ log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}" # Helpers # --------------------------------------------------------------------------- -# Port used by run_bench to check the vLLM server (supervisor for DP>1). -server_health_port() { - local dp="$1" - if [[ "$dp" -gt 1 ]]; then - echo "$VLLM_DP_SUPERVISOR_PORT" - else - echo "$VLLM_PORT" - fi -} - -# Port used by the benchmark client (proxy for DP>1). -bench_port() { - local dp="$1" - if [[ "$dp" -gt 1 ]]; then - echo "$VLLM_DP_PROXY_PORT" - else - echo "$VLLM_PORT" - fi -} - is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -is_proxy_healthy() { - local port="$1" - # The bench client uses /v1/models as the ready signal, so the proxy must - # forward it successfully. - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/v1/models" >/dev/null 2>&1 + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1 } stop_server() { local tp="$1" local dp="$2" - stop_proxy "$tp" "$dp" - local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" if [[ -f "$pid_file" ]]; then local pid @@ -89,74 +59,6 @@ stop_server() { sleep 2 } -start_proxy() { - local tp="$1" - local dp="$2" - if [[ "$dp" -le 1 ]]; then - return 0 - fi - - local proxy_port="$VLLM_DP_PROXY_PORT" - local base_port="$VLLM_PORT" - local backends="" - for ((r = 0; r < dp; r++)); do - backends="${backends:+,}$((base_port + r))" - done - - local pid_file="/data/user1/yy/${EXPERIMENT}_proxy_tp${tp}_dp${dp}.pid" - rm -f "$pid_file" - - log "starting DP proxy on port ${proxy_port} -> backends ${backends}" - nohup "${VENV_VLLM}/bin/python" "${SCRIPT_DIR}/../../scripts/common/dp_proxy.py" \ - --port "$proxy_port" \ - --backends "$backends" \ - --timeout 300 \ - > "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" 2>&1 & - - local pid=$! - echo $pid > "$pid_file" - - for i in $(seq 1 60); do - if is_proxy_healthy "$proxy_port"; then - log "DP proxy is ready on port ${proxy_port}" - return 0 - fi - if ! kill -0 "$pid" 2>/dev/null; then - log "ERROR: DP proxy exited early" - tail -100 "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" - return 1 - fi - sleep 1 - done - - log "ERROR: DP proxy not healthy after 60s" - tail -100 "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" - return 1 -} - -stop_proxy() { - local tp="$1" - local dp="$2" - if [[ "$dp" -le 1 ]]; then - return 0 - fi - - local pid_file="/data/user1/yy/${EXPERIMENT}_proxy_tp${tp}_dp${dp}.pid" - if [[ -f "$pid_file" ]]; then - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "stopping DP proxy pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - fi - pkill -9 -f "dp_proxy.py.*--port ${VLLM_DP_PROXY_PORT}" 2>/dev/null || true - sleep 1 -} - build_server_args() { local tp="$1" local dp="$2" @@ -173,8 +75,6 @@ build_server_args() { if [[ "$dp" -gt 1 ]]; then args+=( --data-parallel-size "$dp" - --data-parallel-size-local "$dp" - --data-parallel-multi-port-external-lb ) fi printf '%s ' "${args[@]}" @@ -188,20 +88,11 @@ start_server() { bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" \ >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 - local hport - hport="$(server_health_port "$dp")" - if ! is_server_healthy "$hport"; then - log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${hport}" + if ! is_server_healthy; then + log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${VLLM_PORT}" return 1 fi - log "vllm server tp=${tp} dp=${dp} is healthy on port ${hport}" - - if [[ "$dp" -gt 1 ]]; then - if ! start_proxy "$tp" "$dp"; then - log "error: failed to start DP proxy for tp=${tp} dp=${dp}" - return 1 - fi - fi + log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}" } run_warmup() { @@ -209,14 +100,12 @@ run_warmup() { local dp="$2" local input_len="$3" local output_len="$4" - local bport - bport="$(bench_port "$dp")" log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)" "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend vllm \ --host 127.0.0.1 \ - --port "$bport" \ + --port "$VLLM_PORT" \ --input-len "$input_len" \ --output-len "$output_len" \ --num 1 \ @@ -393,7 +282,7 @@ run_parallel_config() { "$PYTHON" -m sglang.bench_serving \ --backend vllm \ --host 127.0.0.1 \ - --port "$(bench_port "$dp")" \ + --port "$VLLM_PORT" \ --dataset-name random \ --random-input-len "$isl" \ --random-output-len "$dsl" \ diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh index a4e52e0..b7ec14b 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -8,7 +8,8 @@ # --block-size 256 # --tensor-parallel-size # --no-enable-flashinfer-autotune -# plus DP-related flags when DP > 1. +# When DP > 1, only --data-parallel-size is added so vLLM uses its internal +# DP load balancer (single API server, single port). set -e TP="${1}" @@ -45,20 +46,14 @@ SERVER_ARGS=( if [[ "$DP" -gt 1 ]]; then SERVER_ARGS+=( --data-parallel-size "$DP" - --data-parallel-size-local "$DP" - --data-parallel-multi-port-external-lb ) - # vLLM 0.24.0 hard-codes the DP supervisor port to 9256. - HEALTH_PORT="$VLLM_DP_SUPERVISOR_PORT" -else - HEALTH_PORT="$VLLM_PORT" fi SERVER_ARGS_STR="${SERVER_ARGS[*]}" echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ===" echo "Model: $MODEL_PATH" -echo "Health port: $HEALTH_PORT" +echo "Port: $VLLM_PORT" echo "Command: $SERVER_ARGS_STR" echo "Log: $LOG" @@ -67,11 +62,11 @@ nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & PID=$! echo $PID > "$PID_FILE" echo "PID: $PID" -echo "Waiting for health on port ${HEALTH_PORT}..." +echo "Waiting for health on port ${VLLM_PORT}..." for i in $(seq 1 240); do - if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${HEALTH_PORT}/health" >/dev/null 2>&1; then - echo "vLLM server is ready at http://127.0.0.1:${HEALTH_PORT}" + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" echo "Log: $LOG" exit 0 fi From 0aee540c67fcbc7a60367acdc12e746f6b2edd84 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:50:59 +0000 Subject: [PATCH 20/53] dsv4_h200_vllm_tp_dp_matrix: single server per config, OOM/restart handling, DRY_RUN/GRID_LIMIT, include 1M --- .../dsv4_h200_vllm_tp_dp_matrix/config.env | 7 + .../dsv4_h200_vllm_tp_dp_matrix/matrix.json | 4 +- .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 250 +++++++++++++----- 3 files changed, 197 insertions(+), 64 deletions(-) diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 60a147d..a0aa5ee 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -39,3 +39,10 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" # GPU memory sampling interval (seconds). GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json index 1ed8e9f..539a577 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json @@ -60,8 +60,8 @@ "4096": "N" }, "1048576": { - "128": "N", - "256": "N", + "128": "Y", + "256": "P", "512": "N", "1024": "N", "2048": "N", diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index e3cdaa1..ebd3b85 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -18,6 +18,8 @@ MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" PYTHON="${VENV_CLIENT}/bin/python" @@ -26,7 +28,7 @@ mkdir -p "$log_dir_global" log_init "${log_dir_global}/orchestrator.log" log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" -log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" # --------------------------------------------------------------------------- # Helpers @@ -95,13 +97,20 @@ start_server() { log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}" } -run_warmup() { +restart_server() { local tp="$1" local dp="$2" - local input_len="$3" - local output_len="$4" + log "restarting vllm server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} - log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)" +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend vllm \ --host 127.0.0.1 \ @@ -110,7 +119,7 @@ run_warmup() { --output-len "$output_len" \ --num 1 \ --env-python "$PYTHON" \ - >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.warmup.log" 2>&1 + >> "${log_dir_global}/warmup.log" 2>&1 log "warmup completed" } @@ -136,6 +145,37 @@ except Exception: [[ "${completed:-0}" -ge "$expected" ]] } +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + start_gpu_monitor() { local csv_path="$1" mkdir -p "$(dirname "$csv_path")" @@ -160,7 +200,6 @@ append_scenario_record() { local result_root="$1" local json_path="$result_root/results.json" shift - # Remaining args are key=value pairs. local scenario_json scenario_json="$("$PYTHON" -c " import json, sys @@ -171,6 +210,54 @@ print(json.dumps(d, ensure_ascii=False)) PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" } +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {k: json.loads(v) for k, v in pairs} +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${dsl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + # --------------------------------------------------------------------------- # Per-configuration runner # --------------------------------------------------------------------------- @@ -197,11 +284,6 @@ run_parallel_config() { total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" log "generated ${total_scenarios} scenarios for ${config_label}" - if [[ "$total_scenarios" -eq 0 ]]; then - log "no scenarios for ${config_label}; skipping" - return 0 - fi - # Write metadata. ensure_result_root "$result_root" write_metadata_json \ @@ -218,7 +300,6 @@ run_parallel_config() { "$VENV_VLLM" \ "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" - # Embed static config and the exact server command line used. local server_args_str server_args_str="$(build_server_args "$tp" "$dp")" jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ @@ -232,43 +313,57 @@ run_parallel_config() { }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ mv "${result_root}/results.json.tmp" "${result_root}/results.json" - # Group scenarios by input_len. We restart the server for each ISL to keep - # memory state isolated and to warm up per context length. - local current_isl="" - local group_started=false + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi - tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do - # When the input length changes, restart the server for the new group. - if [[ "$isl" != "$current_isl" ]]; then - if [[ "$group_started" == true ]]; then - stop_server "$tp" "$dp" - fi - current_isl="$isl" + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi - stop_server "$tp" "$dp" - if ! start_server "$tp" "$dp"; then - log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group" - group_started=false - continue - fi - group_started=true + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi - # Warmup with the shortest output for this ISL. - run_warmup "$tp" "$dp" "$isl" 128 + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl dsl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break fi - # If the server for this group did not start, skip all scenarios in it. - if [[ "$group_started" != true ]]; then - log "skipping ${config_label} ISL=${isl} scenario (server not started)" - continue - fi + sname="c${conc}_i${isl}_o${dsl}" + output_file="${raw_dir}/vllm_main_${sname}.jsonl" + detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${sname}.csv" - local output_file="${raw_dir}/vllm_main_c${conc}_i${isl}_o${dsl}.jsonl" - local detail_log="${phase_log_dir}/vllm_${config_label}_c${conc}_i${isl}_o${dsl}.log" - local gpu_csv="${gpu_log_dir}/gpu_mem_c${conc}_i${isl}_o${dsl}.csv" - - if scenario_already_completed "$output_file" "$num"; then - log "skipping already-completed ${config_label} scenario: c=${conc} i=${isl} o=${dsl}" + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" continue fi @@ -277,7 +372,7 @@ run_parallel_config() { local gpu_pid gpu_pid="$(start_gpu_monitor "$gpu_csv")" - local bench_rc=0 + bench_rc=0 timeout "$SCENARIO_TIMEOUT_S" \ "$PYTHON" -m sglang.bench_serving \ --backend vllm \ @@ -295,25 +390,56 @@ run_parallel_config() { stop_gpu_monitor "$gpu_pid" - if [[ "$bench_rc" -ne 0 ]]; then - log "ERROR: ${config_label} scenario c=${conc} i=${isl} o=${dsl} failed (rc=${bench_rc}); see ${detail_log}" - if [[ "$mark" == "P" ]]; then - log "optional (P) scenario failed; recording as skipped_oom and continuing" - append_scenario_record "$result_root" \ - "name=c${conc}_i${isl}_o${dsl}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ - "status=\"skipped_oom\"" \ - "note=\"bench command failed or timed out (rc=${bench_rc})\"" - continue - else - log "mandatory (Y) scenario failed; aborting current ISL group" - stop_server "$tp" "$dp" - group_started=false - continue - fi + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + continue fi - log "finished ${config_label} scenario: output=${output_file}" + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break done stop_server "$tp" "$dp" From 0ae1ea0a9f3c722f93a14e191ed06f81f968ea09 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 12:33:52 +0000 Subject: [PATCH 21/53] dsv4_h200_vllm_tp_dp_matrix: fix output naming, parser/compat, add Docker startup option --- .../dsv4_h200_vllm_tp_dp_matrix/compare.py | 7 +- .../dsv4_h200_vllm_tp_dp_matrix/config.env | 5 + .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 23 ++++- .../start_vllm_docker.sh | 94 +++++++++++++++++++ .../start_vllm_dp.sh | 4 + 5 files changed, 126 insertions(+), 7 deletions(-) create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py index 8bf26df..bdd3224 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py @@ -6,6 +6,7 @@ Usage: """ import argparse import json +import re from collections import defaultdict from pathlib import Path @@ -106,8 +107,8 @@ def main(): f.write("| " + " | ".join(headers) + " |\n") f.write("|" + "|".join(["---"] * len(headers)) + "|\n") - for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, x.replace("c", "").replace("i", "_").replace("o", "_").split("_")[1:]))): - cfg_part, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, dsl = re.findall(r"\d+", scenario_name) # cfg_part not used; just for readability. for label, data in configs: s = by_scenario[scenario_name].get(label) @@ -134,7 +135,7 @@ def main(): f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} for scenario_name, backends in by_scenario.items(): - _, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + _, isl, dsl = re.findall(r"\d+", scenario_name) isl_i, dsl_i = int(isl), int(dsl) for label, s in backends.items(): m = s["metrics"] diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index a0aa5ee..1652629 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -46,3 +46,8 @@ DRY_RUN="${DRY_RUN:-0}" # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. GRID_LIMIT="${GRID_LIMIT:-0}" + +# Docker deployment switch. If 1, vLLM is launched via Docker using the image +# below instead of the local VENV_VLLM environment. +USE_DOCKER="${USE_DOCKER:-0}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index ebd3b85..99e8de3 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -204,7 +204,12 @@ append_scenario_record() { scenario_json="$("$PYTHON" -c " import json, sys pairs = [a.split('=', 1) for a in sys.argv[1:]] -d = {k: json.loads(v) for k, v in pairs} +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v print(json.dumps(d, ensure_ascii=False)) " "$@")" PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" @@ -218,7 +223,12 @@ record_skipped_csv() { row="$("$PYTHON" -c " import csv, json, sys, io pairs = [a.split('=', 1) for a in sys.argv[1:]] -d = {k: json.loads(v) for k, v in pairs} +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v buf = io.StringIO() writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore') writer.writerow(d) @@ -358,9 +368,9 @@ run_parallel_config() { fi sname="c${conc}_i${isl}_o${dsl}" - output_file="${raw_dir}/vllm_main_${sname}.jsonl" + output_file="${raw_dir}/vllm_main_${conc}_${isl}_${dsl}.jsonl" detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log" - gpu_csv="${gpu_log_dir}/gpu_mem_${sname}.csv" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${dsl}.csv" if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then log "skipping already-processed ${config_label} scenario: ${sname}" @@ -392,6 +402,11 @@ run_parallel_config() { if [[ "$bench_rc" -eq 0 ]]; then log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" continue fi diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh new file mode 100755 index 0000000..ae7f728 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Start vLLM server in Docker for a given TPxDP configuration. +# Usage: start_vllm_docker.sh +# +# Uses the vllm/vllm-openai image and the same minimal argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +PORT="${VLLM_PORT:-30030}" +NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" +PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype bfloat16 + --block-size 256 + --tensor-parallel-size "$TP" + --no-enable-flashinfer-autotune + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: vllm $SERVER_ARGS_STR" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --entrypoint vllm \ + -p "${PORT}:${PORT}" \ + -v /data/models:/data/models:ro \ + -v /data/user1/yy/tmp:/tmp \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh index b7ec14b..77b662a 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -19,6 +19,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" +if [[ "${USE_DOCKER:-0}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" +fi + cd /data/user1/yy mkdir -p logs From 8d61e682cd9c6c0f77f18ac7a95e4285ee13ba39 Mon Sep 17 00:00:00 2001 From: Root User Date: Thu, 9 Jul 2026 14:49:31 +0000 Subject: [PATCH 22/53] =?UTF-8?q?add=20dsv4=5Fh200=5Fsglang=5Ftp=5Fdp=5Fma?= =?UTF-8?q?trix=20experiment=20(Docker-based=20SGLang=20TP=C3=97DP=20matri?= =?UTF-8?q?x)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dsv4_h200_sglang_tp_dp_matrix/compare.py | 162 ++++++ .../dsv4_h200_sglang_tp_dp_matrix/config.env | 62 +++ .../generate_scenarios.py | 105 ++++ .../dsv4_h200_sglang_tp_dp_matrix/matrix.json | 81 +++ .../run_bench.sh | 527 ++++++++++++++++++ .../smoke_matrix.json | 18 + .../start_sglang_docker.sh | 96 ++++ .../start_sglang_dp.sh | 85 +++ 8 files changed, 1136 insertions(+) create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/compare.py create mode 100644 experiments/dsv4_h200_sglang_tp_dp_matrix/config.env create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh create mode 100644 experiments/dsv4_h200_sglang_tp_dp_matrix/smoke_matrix.json create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/compare.py b/experiments/dsv4_h200_sglang_tp_dp_matrix/compare.py new file mode 100755 index 0000000..2029e7b --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h200_sglang_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: SGLang (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, dsl = re.findall(r"\d+", scenario_name) + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/DSL. + f.write("\n## Best throughput per (ISL, DSL)\n\n") + f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, dsl = re.findall(r"\d+", scenario_name) + isl_i, dsl_i = int(isl), int(dsl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]: + best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s) + for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env new file mode 100644 index 0000000..3f2bf08 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env @@ -0,0 +1,62 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using SGLang Docker. +# Tests SGLang with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_h200_sglang_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +SGLANG_PORT="${SGLANG_PORT:-30031}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" + +# When USE_DOCKER_CLIENT=1, the benchmark client (sglang.bench_serving) is run +# inside the same SGLang Docker image used for the server. This avoids requiring +# a native sglang installation on the host. +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# SGLang server settings +CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" +MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" + +# Docker deployment switch. SGLang is launched via Docker using the image below. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. 0 means use the default heuristic in +# generate_scenarios.py (6-8 points, or all integers when range is small). +CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py new file mode 100755 index 0000000..ef46e01 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return a sorted list of concurrency values in [low, high]. + + If the range is small, return every integer. Otherwise sample roughly + `target` points linearly between low and high (inclusive). + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if high - low + 1 <= target: + return list(range(low, high + 1)) + + points = set() + points.add(low) + points.add(high) + step = (high - low) / (target - 1) + for i in range(1, target - 1): + v = low + round(step * i) + points.add(max(low, min(high, v))) + return sorted(points) + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + dsl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for dsl_str in sorted(dsl_map.keys(), key=int): + mark = dsl_map[dsl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(dsl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..55dcf58 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json @@ -0,0 +1,81 @@ +{ + "comment": "ISL/DSL matrix for dsv4_h200_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 }, + "262144": { "low": 1, "high": 2 }, + "524288": { "low": 1, "high": 2 }, + "1048576": { "low": 1, "high": 2 } + } +} diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..13cf241 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -0,0 +1,527 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash on SGLang (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + local pid_file="/data/user1/yy/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + + # Fallback: remove any Docker container started by this experiment. + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + + # Fallback: kill any sglang serve processes for this model. + pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "sglang serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + + local args=( + "sglang serve" "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --dp-size "$dp" + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + if ! is_server_healthy; then + log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}" + return 1 + fi + log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + docker run --rm \ + --network host \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RESULT_BASE}:${RESULT_BASE}" \ + -e PYTHONUNBUFFERED=1 \ + "${DOCKER_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$SGLANG_PORT" \ + --dataset-name random \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "${log_dir_global}/warmup.log" 2>&1 + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${dsl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "sglang", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_sglang_dp.sh", + "server_args": $args + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl dsl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${dsl}" + output_file="${raw_dir}/sglang_main_${conc}_${isl}_${dsl}.jsonl" + detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${dsl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" \ + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$SGLANG_PORT" \ + --dataset-name random \ + --random-input-len "$isl" \ + --random-output-len "$dsl" \ + --num-prompts "$num" \ + --max-concurrency "$conc" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/smoke_matrix.json b/experiments/dsv4_h200_sglang_tp_dp_matrix/smoke_matrix.json new file mode 100644 index 0000000..5715c28 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/smoke_matrix.json @@ -0,0 +1,18 @@ +{ + "comment": "Small smoke matrix for quickly verifying TP×DP server startup and bench path.", + "mode": "Y", + "matrix": { + "1024": { + "128": "Y", + "256": "Y" + }, + "4096": { + "128": "Y", + "256": "Y" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 8 }, + "4096": { "low": 1, "high": 4 } + } +} diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh new file mode 100755 index 0000000..4e1ca53 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Start SGLang server in Docker for a given TPxDP configuration. +# Usage: start_sglang_docker.sh +# +# Uses the lmsysorg/sglang image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" +PID_FILE="/data/user1/yy/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --entrypoint sglang \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v /data/user1/yy/tmp:/tmp \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh new file mode 100755 index 0000000..9a47e07 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the +# local VENV_CLIENT environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" +fi + +cd /data/user1/yy +mkdir -p logs + +VENV="${VENV_CLIENT}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + sglang serve "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${SGLANG_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 From 31731ebb758eb140e8c060e1dae87b336a27deda Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 15:23:57 +0000 Subject: [PATCH 23/53] dsv4_h200_vllm_tp_dp_matrix: export CONCURRENCY_SAMPLES so generate_scenarios.py sees it --- experiments/dsv4_h200_vllm_tp_dp_matrix/config.env | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 1652629..61b8be2 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -30,9 +30,10 @@ declare -a PARALLEL_CONFIGS=( MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" -# Sampling density for concurrency. 0 means use the default heuristic in -# generate_scenarios.py (6-8 points, or all integers when range is small). -CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}" +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the two endpoints (low and high). +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" # Per-scenario timeout to avoid hangs (seconds). SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" From db419e17b855914d835361850df8a339461f5070 Mon Sep 17 00:00:00 2001 From: Root User Date: Thu, 9 Jul 2026 15:36:03 +0000 Subject: [PATCH 24/53] dsv4_h200_sglang_tp_dp_matrix: sample only low/high concurrency, use RUNTIME_BASE for self-contained runtime files --- .../dsv4_h200_sglang_tp_dp_matrix/config.env | 4 ++++ .../generate_scenarios.py | 21 +++++++------------ .../run_bench.sh | 2 +- .../start_sglang_docker.sh | 10 ++++----- .../start_sglang_dp.sh | 10 ++++----- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env index 3f2bf08..600b4cd 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env @@ -23,6 +23,10 @@ USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + # Parallel configurations to test. Format: "TP DP" declare -a PARALLEL_CONFIGS=( "2 4" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py index ef46e01..f722479 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/generate_scenarios.py @@ -15,23 +15,16 @@ from pathlib import Path def sample_concurrency(low: int, high: int, target: int) -> list[int]: - """Return a sorted list of concurrency values in [low, high]. + """Return only the low and high concurrency values in [low, high]. - If the range is small, return every integer. Otherwise sample roughly - `target` points linearly between low and high (inclusive). + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. """ assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" - if high - low + 1 <= target: - return list(range(low, high + 1)) - - points = set() - points.add(low) - points.add(high) - step = (high - low) / (target - 1) - for i in range(1, target - 1): - v = low + round(step * i) - points.add(max(low, min(high, v))) - return sorted(points) + if low == high: + return [low] + return [low, high] def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 13cf241..75713d0 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -48,7 +48,7 @@ stop_server() { local tp="$1" local dp="$2" - local pid_file="/data/user1/yy/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" if [[ -f "$pid_file" ]]; then local pid pid="$(cat "$pid_file")" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh index 4e1ca53..4ddd72f 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -13,15 +13,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -cd /data/user1/yy -mkdir -p logs +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" PORT="${SGLANG_PORT:-30031}" NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" -PID_FILE="/data/user1/yy/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" -LOG="/data/user1/yy/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" rm -f "$PID_FILE" # Clean up any stale container with the same name. @@ -64,7 +64,7 @@ nohup docker run --rm \ --entrypoint sglang \ -p "${PORT}:${PORT}" \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ - -v /data/user1/yy/tmp:/tmp \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ "$IMAGE" \ diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh index 9a47e07..35d52fe 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -18,17 +18,17 @@ if [[ "${USE_DOCKER:-1}" == "1" ]]; then exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" fi -cd /data/user1/yy -mkdir -p logs +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" VENV="${VENV_CLIENT}" export PATH="$VENV/bin:$PATH" export PYTHONUNBUFFERED=1 -export TMPDIR=/data/user1/yy/tmp +export TMPDIR="${RUNTIME_BASE}/tmp" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" -LOG="/data/user1/yy/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" From 7852dd20404e4962bbc321788f387c383a9f2d43 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:17:00 +0000 Subject: [PATCH 25/53] sglang tp_dp_matrix: run SGLang server/client in Docker on H200 - config.env: switch to /data/models/DeepSeek-V4-Flash, USE_DOCKER=1, CONTEXT_LENGTH=1M, CONCURRENCY_SAMPLES=2, add DATASET_PATH. - start_sglang_docker.sh: add --ipc host --shm-size 16g for NCCL. - start_sglang_dp.sh: use --model-path for Docker CLI. - run_bench.sh: mount dataset into client container and pass --dataset-path to avoid HF download on offline nodes. --- .../dsv4_h200_sglang_tp_dp_matrix/config.env | 26 +++++++++++-------- .../run_bench.sh | 3 +++ .../start_sglang_docker.sh | 5 +++- .../start_sglang_dp.sh | 3 ++- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env index 600b4cd..d5ac289 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env @@ -1,4 +1,4 @@ -# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using SGLang Docker. +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using SGLang. # Tests SGLang with three parallel configurations: # TP=2, DP=4 -> 2 GPUs per replica, 4 replicas # TP=4, DP=2 -> 4 GPUs per replica, 2 replicas @@ -6,7 +6,7 @@ EXPERIMENT="dsv4_h200_sglang_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30031}" @@ -16,9 +16,7 @@ SGLANG_PORT="${SGLANG_PORT:-30031}" # does not exist on the host. VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# When USE_DOCKER_CLIENT=1, the benchmark client (sglang.bench_serving) is run -# inside the same SGLang Docker image used for the server. This avoids requiring -# a native sglang installation on the host. +# Run the benchmark client natively (0) or inside Docker (1). USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" @@ -35,22 +33,28 @@ declare -a PARALLEL_CONFIGS=( ) # SGLang server settings -CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" +CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" -# Docker deployment switch. SGLang is launched via Docker using the image below. +# Deployment switch. 0 = native sglang venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H200 nodes. +DATASET_PATH="${DATASET_PATH:-/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json}" + # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" -# Sampling density for concurrency. 0 means use the default heuristic in -# generate_scenarios.py (6-8 points, or all integers when range is small). -CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}" +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" # Per-scenario timeout to avoid hangs (seconds). SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 75713d0..09a5e09 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -123,6 +123,7 @@ run_bench_serving() { docker run --rm \ --network host \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${DATASET_PATH}:${DATASET_PATH}:ro" \ -v "${RESULT_BASE}:${RESULT_BASE}" \ -e PYTHONUNBUFFERED=1 \ "${DOCKER_IMAGE}" \ @@ -142,6 +143,7 @@ run_warmup() { --host 127.0.0.1 \ --port "$SGLANG_PORT" \ --dataset-name random \ + --dataset-path "$DATASET_PATH" \ --random-input-len "$input_len" \ --random-output-len "$output_len" \ --num-prompts 1 \ @@ -419,6 +421,7 @@ run_parallel_config() { --host 127.0.0.1 \ --port "$SGLANG_PORT" \ --dataset-name random \ + --dataset-path "$DATASET_PATH" \ --random-input-len "$isl" \ --random-output-len "$dsl" \ --num-prompts "$num" \ diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh index 4ddd72f..5753e46 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -28,7 +28,8 @@ rm -f "$PID_FILE" docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( - serve "$MODEL_PATH" + serve + --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" @@ -61,6 +62,8 @@ echo "Log: $LOG" nohup docker run --rm \ --name "$NAME" \ --gpus all \ + --ipc host \ + --shm-size 16g \ --entrypoint sglang \ -p "${PORT}:${PORT}" \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh index 35d52fe..230bf88 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -33,7 +33,8 @@ PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" SERVER_ARGS=( - sglang serve "$MODEL_PATH" + sglang serve + --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" From f7f4e928d95899d8dca03241c767105fb370eafe Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:17:38 +0000 Subject: [PATCH 26/53] sglang tp_dp_matrix: exclude 1M ISL from mandatory matrix per current scope --- experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json index 55dcf58..c3c3534 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json @@ -60,8 +60,8 @@ "4096": "N" }, "1048576": { - "128": "Y", - "256": "P", + "128": "N", + "256": "N", "512": "N", "1024": "N", "2048": "N", From 06cfab20d223f766692b7952cdac69375447a174 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:18:34 +0000 Subject: [PATCH 27/53] sglang tp_dp_matrix: keep 1M ISL (128 Y, 256 P) in scope --- experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json index c3c3534..55dcf58 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/matrix.json @@ -60,8 +60,8 @@ "4096": "N" }, "1048576": { - "128": "N", - "256": "N", + "128": "Y", + "256": "P", "512": "N", "1024": "N", "2048": "N", From 666c58384413f41e03c241fcf8fe9e100a8715c7 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:20:36 +0000 Subject: [PATCH 28/53] vllm tp_dp_matrix: add final results (report, results.json, comparison, scenarios) Source code was already committed; this adds only the structured final artifacts from run 20260709-152419_matrix. Raw jsonl/logs/gpu_logs remain ignored per .gitignore. --- .../20260709-152419_matrix/comparison.md | 230 ++ .../skipped_after_oom.csv | 1 + .../20260709-152419_matrix/tp2_dp4/report.md | 70 + .../tp2_dp4/results.json | 3500 +++++++++++++++++ .../tp2_dp4/scenarios.tsv | 57 + .../20260709-152419_matrix/tp4_dp2/report.md | 70 + .../tp4_dp2/results.json | 3500 +++++++++++++++++ .../tp4_dp2/scenarios.tsv | 57 + .../20260709-152419_matrix/tp8_dp1/report.md | 70 + .../tp8_dp1/results.json | 3500 +++++++++++++++++ .../tp8_dp1/scenarios.tsv | 57 + 11 files changed, 11112 insertions(+) create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/comparison.md create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/skipped_after_oom.csv create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/report.md create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/results.json create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/scenarios.tsv create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/report.md create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/results.json create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/scenarios.tsv create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/report.md create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/results.json create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/scenarios.tsv diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/comparison.md b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/comparison.md new file mode 100644 index 0000000..7922279 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/comparison.md @@ -0,0 +1,230 @@ +# vLLM TP×DP matrix comparison (8x NVIDIA H200 143GB) + +## Summary + +- Model: `/data/models/DeepSeek-V4-Flash` +- Hardware: 8x NVIDIA H200 143GB +- Backend: vLLM +- Benchmark client: `sglang.bench_serving` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +### Configurations + +| Config | TP | DP | GPUs/replica | Notes | +|---|---:|---:|---:|---| +| tp2_dp4 | 2 | 4 | 2 | server args recorded per ISL in results.json | +| tp4_dp2 | 4 | 2 | 4 | server args recorded per ISL in results.json | +| tp8_dp1 | 8 | 1 | 8 | server args recorded per ISL in results.json | + +## Side-by-side results + +| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| c1_i1024_o128 | 1024 | 128 | tp2_dp4 | 1 | 1.05 | 79.65 | 319.08 | 355.57 | 10.56 | 10.99 | 11.00 | 1406.84 | 138264/143771 (96.2%) | ✅ | +| c1_i1024_o128 | 1024 | 128 | tp4_dp2 | 1 | 1.34 | 101.85 | 302.85 | 345.83 | 8.17 | 8.32 | 8.35 | 1055.00 | 138396/143771 (96.3%) | ✅ | +| c1_i1024_o128 | 1024 | 128 | tp8_dp1 | 1 | 1.61 | 121.80 | 274.43 | 314.97 | 6.74 | 6.78 | 6.79 | 871.45 | 131900/143771 (91.7%) | ✅ | +| c1_i1024_o256 | 1024 | 256 | tp2_dp4 | 1 | 0.84 | 84.85 | 107.93 | 108.38 | 11.01 | 11.57 | 11.61 | 2082.56 | 139322/143771 (96.9%) | ✅ | +| c1_i1024_o256 | 1024 | 256 | tp4_dp2 | 1 | 1.12 | 113.80 | 88.52 | 90.59 | 8.12 | 8.21 | 8.23 | 1572.36 | 139454/143771 (97.0%) | ✅ | +| c1_i1024_o256 | 1024 | 256 | tp8_dp1 | 1 | 1.40 | 141.77 | 44.63 | 45.58 | 6.71 | 6.72 | 6.72 | 1281.54 | 132930/143771 (92.5%) | ✅ | +| c1_i1024_o512 | 1024 | 512 | tp2_dp4 | 1 | 0.79 | 79.92 | 113.71 | 115.02 | 11.50 | 11.85 | 11.86 | 2303.60 | 139322/143771 (96.9%) | ✅ | +| c1_i1024_o512 | 1024 | 512 | tp4_dp2 | 1 | 1.09 | 110.67 | 187.00 | 208.64 | 8.12 | 8.17 | 8.18 | 1572.17 | 139454/143771 (97.0%) | ✅ | +| c1_i1024_o512 | 1024 | 512 | tp8_dp1 | 1 | 1.36 | 137.55 | 65.04 | 65.69 | 6.74 | 6.78 | 6.79 | 1305.74 | 132930/143771 (92.5%) | ✅ | +| c1_i1024_o1024 | 1024 | 1024 | tp2_dp4 | 1 | 0.27 | 83.58 | 100.14 | 100.52 | 11.71 | 11.90 | 11.93 | 8200.76 | 139322/143771 (96.9%) | ✅ | +| c1_i1024_o1024 | 1024 | 1024 | tp4_dp2 | 1 | 0.39 | 120.49 | 90.47 | 90.97 | 8.11 | 8.19 | 8.21 | 5705.34 | 139456/143771 (97.0%) | ✅ | +| c1_i1024_o1024 | 1024 | 1024 | tp8_dp1 | 1 | 0.48 | 146.08 | 60.67 | 60.68 | 6.71 | 6.73 | 6.73 | 4727.62 | 132932/143771 (92.5%) | ✅ | +| c1_i1024_o2048 | 1024 | 2048 | tp2_dp4 | 1 | 0.08 | 84.74 | 105.11 | 105.45 | 11.70 | 11.81 | 11.81 | 20192.89 | 139322/143771 (96.9%) | ✅ | +| c1_i1024_o2048 | 1024 | 2048 | tp4_dp2 | 1 | 0.11 | 122.91 | 84.75 | 84.77 | 8.08 | 8.09 | 8.09 | 13978.61 | 139456/143771 (97.0%) | ✅ | +| c1_i1024_o2048 | 1024 | 2048 | tp8_dp1 | 1 | 0.13 | 148.32 | 60.19 | 62.81 | 6.71 | 6.72 | 6.72 | 11586.89 | 132932/143771 (92.5%) | ✅ | +| c1_i1024_o4096 | 1024 | 4096 | tp2_dp4 | 1 | 0.04 | 86.04 | 114.24 | 116.37 | 11.59 | 11.84 | 11.86 | 43455.08 | 139322/143771 (96.9%) | ✅ | +| c1_i1024_o4096 | 1024 | 4096 | tp4_dp2 | 1 | 0.05 | 122.97 | 82.10 | 82.14 | 8.09 | 8.12 | 8.13 | 30482.21 | 139456/143771 (97.0%) | ✅ | +| c1_i1024_o4096 | 1024 | 4096 | tp8_dp1 | 1 | 0.06 | 148.14 | 46.60 | 47.48 | 6.73 | 6.76 | 6.76 | 25332.98 | 132932/143771 (92.5%) | ✅ | +| c1_i4096_o128 | 4096 | 128 | tp2_dp4 | 1 | 0.92 | 69.56 | 353.45 | 371.20 | 11.63 | 11.95 | 12.00 | 1608.68 | 139322/143771 (96.9%) | ✅ | +| c1_i4096_o128 | 4096 | 128 | tp4_dp2 | 1 | 1.44 | 109.20 | 96.00 | 96.62 | 8.16 | 8.23 | 8.24 | 1054.23 | 139456/143771 (97.0%) | ✅ | +| c1_i4096_o128 | 4096 | 128 | tp8_dp1 | 1 | 1.62 | 122.84 | 148.51 | 155.20 | 6.72 | 6.74 | 6.74 | 918.13 | 132932/143771 (92.5%) | ✅ | +| c1_i4096_o256 | 4096 | 256 | tp2_dp4 | 1 | 0.77 | 78.01 | 208.90 | 226.63 | 11.61 | 11.87 | 11.89 | 2296.91 | 139324/143771 (96.9%) | ✅ | +| c1_i4096_o256 | 4096 | 256 | tp4_dp2 | 1 | 1.11 | 112.37 | 91.50 | 91.90 | 8.16 | 8.23 | 8.24 | 1604.71 | 139456/143771 (97.0%) | ✅ | +| c1_i4096_o256 | 4096 | 256 | tp8_dp1 | 1 | 1.37 | 138.57 | 67.18 | 67.54 | 6.73 | 6.77 | 6.78 | 1320.30 | 132932/143771 (92.5%) | ✅ | +| c1_i4096_o512 | 4096 | 512 | tp2_dp4 | 1 | 0.77 | 78.30 | 210.04 | 232.13 | 11.63 | 11.95 | 11.98 | 2378.47 | 139324/143771 (96.9%) | ✅ | +| c1_i4096_o512 | 4096 | 512 | tp4_dp2 | 1 | 1.10 | 111.43 | 104.51 | 107.11 | 8.16 | 8.24 | 8.25 | 1621.06 | 139456/143771 (97.0%) | ✅ | +| c1_i4096_o512 | 4096 | 512 | tp8_dp1 | 1 | 1.37 | 138.55 | 64.63 | 65.10 | 6.73 | 6.78 | 6.78 | 1320.67 | 132932/143771 (92.5%) | ✅ | +| c1_i4096_o1024 | 4096 | 1024 | tp2_dp4 | 1 | 0.27 | 83.60 | 113.91 | 115.64 | 11.55 | 11.86 | 11.87 | 8360.74 | 139324/143771 (96.9%) | ✅ | +| c1_i4096_o1024 | 4096 | 1024 | tp4_dp2 | 1 | 0.39 | 119.41 | 92.40 | 93.46 | 8.15 | 8.23 | 8.24 | 5771.30 | 139456/143771 (97.0%) | ✅ | +| c1_i4096_o1024 | 4096 | 1024 | tp8_dp1 | 1 | 0.47 | 144.76 | 66.58 | 66.81 | 6.73 | 6.78 | 6.79 | 4791.74 | 132932/143771 (92.5%) | ✅ | +| c1_i4096_o2048 | 4096 | 2048 | tp2_dp4 | 1 | 0.08 | 86.30 | 122.31 | 124.03 | 11.54 | 11.75 | 11.78 | 19528.19 | 139324/143771 (96.9%) | ✅ | +| c1_i4096_o2048 | 4096 | 2048 | tp4_dp2 | 1 | 0.11 | 121.84 | 105.61 | 108.34 | 8.13 | 8.16 | 8.16 | 14152.06 | 139824/143771 (97.3%) | ✅ | +| c1_i4096_o2048 | 4096 | 2048 | tp8_dp1 | 1 | 0.13 | 146.91 | 67.22 | 67.42 | 6.75 | 6.79 | 6.80 | 11755.04 | 132932/143771 (92.5%) | ✅ | +| c1_i4096_o4096 | 4096 | 4096 | tp2_dp4 | 1 | 0.04 | 85.60 | 130.43 | 132.68 | 11.66 | 11.87 | 11.89 | 43588.26 | 139324/143771 (96.9%) | ✅ | +| c1_i4096_o4096 | 4096 | 4096 | tp4_dp2 | 1 | 0.05 | 122.26 | 110.67 | 114.17 | 8.14 | 8.17 | 8.17 | 30710.72 | 139824/143771 (97.3%) | ✅ | +| c1_i4096_o4096 | 4096 | 4096 | tp8_dp1 | 1 | 0.06 | 147.18 | 66.70 | 66.85 | 6.77 | 6.80 | 6.80 | 25525.53 | 132932/143771 (92.5%) | ✅ | +| c1_i16384_o128 | 16384 | 128 | tp2_dp4 | 1 | 0.88 | 66.35 | 599.10 | 618.22 | 11.00 | 11.48 | 11.54 | 1948.72 | 139324/143771 (96.9%) | ✅ | +| c1_i16384_o128 | 16384 | 128 | tp4_dp2 | 1 | 1.14 | 86.24 | 512.76 | 518.98 | 8.11 | 8.23 | 8.25 | 1441.62 | 139824/143771 (97.3%) | ✅ | +| c1_i16384_o128 | 16384 | 128 | tp8_dp1 | 1 | 1.34 | 101.29 | 433.34 | 440.01 | 6.72 | 6.80 | 6.80 | 1242.36 | 132932/143771 (92.5%) | ✅ | +| c1_i16384_o256 | 16384 | 256 | tp2_dp4 | 1 | 0.71 | 71.69 | 448.96 | 486.54 | 11.52 | 11.66 | 11.67 | 2621.34 | 139324/143771 (96.9%) | ✅ | +| c1_i16384_o256 | 16384 | 256 | tp4_dp2 | 1 | 0.98 | 99.42 | 407.06 | 440.10 | 8.14 | 8.23 | 8.24 | 1937.46 | 139824/143771 (97.3%) | ✅ | +| c1_i16384_o256 | 16384 | 256 | tp8_dp1 | 1 | 1.31 | 132.55 | 122.35 | 122.83 | 6.73 | 6.76 | 6.76 | 1372.37 | 132932/143771 (92.5%) | ✅ | +| c1_i16384_o512 | 16384 | 512 | tp2_dp4 | 1 | 0.77 | 77.70 | 188.54 | 190.96 | 11.55 | 11.73 | 11.74 | 2356.50 | 139684/143771 (97.2%) | ✅ | +| c1_i16384_o512 | 16384 | 512 | tp4_dp2 | 1 | 1.07 | 108.69 | 155.92 | 157.11 | 8.16 | 8.23 | 8.25 | 1661.52 | 140338/143771 (97.6%) | ✅ | +| c1_i16384_o512 | 16384 | 512 | tp8_dp1 | 1 | 1.31 | 132.98 | 118.65 | 119.23 | 6.73 | 6.76 | 6.76 | 1365.21 | 132932/143771 (92.5%) | ✅ | +| c1_i16384_o1024 | 16384 | 1024 | tp2_dp4 | 1 | 0.27 | 83.88 | 322.92 | 349.35 | 11.31 | 11.64 | 11.64 | 7915.88 | 139684/143771 (97.2%) | ✅ | +| c1_i16384_o1024 | 16384 | 1024 | tp4_dp2 | 1 | 0.38 | 117.41 | 158.31 | 161.09 | 8.16 | 8.23 | 8.24 | 5837.98 | 140854/143771 (98.0%) | ✅ | +| c1_i16384_o1024 | 16384 | 1024 | tp8_dp1 | 1 | 0.46 | 141.13 | 132.08 | 134.06 | 6.75 | 6.80 | 6.81 | 4873.19 | 132932/143771 (92.5%) | ✅ | +| c1_i16384_o2048 | 16384 | 2048 | tp2_dp4 | 1 | 0.08 | 85.62 | 249.46 | 261.08 | 11.54 | 11.64 | 11.65 | 20210.50 | 139684/143771 (97.2%) | ✅ | +| c1_i16384_o2048 | 16384 | 2048 | tp4_dp2 | 1 | 0.11 | 121.16 | 155.57 | 156.44 | 8.14 | 8.18 | 8.19 | 14233.84 | 140854/143771 (98.0%) | ✅ | +| c1_i16384_o2048 | 16384 | 2048 | tp8_dp1 | 1 | 0.13 | 145.70 | 132.65 | 135.00 | 6.77 | 6.82 | 6.82 | 11867.20 | 132932/143771 (92.5%) | ✅ | +| c1_i65536_o128 | 65536 | 128 | tp2_dp4 | 1 | 0.44 | 33.25 | 2811.65 | 2892.88 | 11.47 | 12.08 | 12.21 | 4233.91 | 139706/143771 (97.2%) | ✅ | +| c1_i65536_o128 | 65536 | 128 | tp4_dp2 | 1 | 0.56 | 42.54 | 2211.56 | 2223.60 | 7.89 | 8.16 | 8.19 | 3092.42 | 140854/143771 (98.0%) | ✅ | +| c1_i65536_o128 | 65536 | 128 | tp8_dp1 | 1 | 0.84 | 63.38 | 1783.81 | 1903.90 | 6.67 | 6.75 | 6.75 | 2686.50 | 133260/143771 (92.7%) | ✅ | +| c1_i65536_o256 | 65536 | 256 | tp2_dp4 | 1 | 0.48 | 48.96 | 2181.49 | 2314.91 | 11.55 | 11.95 | 11.96 | 3709.96 | 139708/143771 (97.2%) | ✅ | +| c1_i65536_o256 | 65536 | 256 | tp4_dp2 | 1 | 1.01 | 102.18 | 274.65 | 283.52 | 7.97 | 8.23 | 8.25 | 1647.86 | 140854/143771 (98.0%) | ✅ | +| c1_i65536_o256 | 65536 | 256 | tp8_dp1 | 1 | 1.19 | 120.56 | 254.47 | 256.20 | 6.74 | 6.79 | 6.80 | 1352.80 | 133780/143771 (93.1%) | ✅ | +| c1_i65536_o512 | 65536 | 512 | tp2_dp4 | 1 | 0.62 | 63.19 | 1276.37 | 1453.96 | 11.06 | 11.68 | 11.78 | 2624.42 | 139708/143771 (97.2%) | ✅ | +| c1_i65536_o512 | 65536 | 512 | tp4_dp2 | 1 | 0.98 | 99.45 | 326.62 | 330.25 | 7.91 | 8.17 | 8.18 | 1655.09 | 140854/143771 (98.0%) | ✅ | +| c1_i65536_o512 | 65536 | 512 | tp8_dp1 | 1 | 1.18 | 119.95 | 288.97 | 290.20 | 6.54 | 6.74 | 6.75 | 1356.99 | 133780/143771 (93.1%) | ✅ | +| c1_i65536_o1024 | 65536 | 1024 | tp2_dp4 | 1 | 0.27 | 83.13 | 375.58 | 383.90 | 11.12 | 11.57 | 11.58 | 8213.89 | 139708/143771 (97.2%) | ✅ | +| c1_i65536_o1024 | 65536 | 1024 | tp4_dp2 | 1 | 0.37 | 112.31 | 358.50 | 364.11 | 8.07 | 8.26 | 8.26 | 5842.34 | 140854/143771 (98.0%) | ✅ | +| c1_i65536_o1024 | 65536 | 1024 | tp8_dp1 | 1 | 0.45 | 136.44 | 254.42 | 254.84 | 6.76 | 6.85 | 6.86 | 4841.93 | 133780/143771 (93.1%) | ✅ | +| c1_i131072_o128 | 131072 | 128 | tp2_dp4 | 1 | 0.32 | 24.40 | 3831.71 | 3847.18 | 11.06 | 11.65 | 11.68 | 5071.55 | 140228/143771 (97.5%) | ⚠️ | +| c1_i131072_o128 | 131072 | 128 | tp4_dp2 | 1 | 0.38 | 28.62 | 3498.43 | 3510.98 | 7.74 | 8.18 | 8.22 | 4353.06 | 141374/143771 (98.3%) | ⚠️ | +| c1_i131072_o128 | 131072 | 128 | tp8_dp1 | 1 | 0.55 | 42.03 | 3042.40 | 3072.01 | 6.34 | 6.70 | 6.72 | 3832.09 | 133780/143771 (93.1%) | ⚠️ | +| c1_i131072_o256 | 131072 | 256 | tp2_dp4 | 1 | 0.46 | 46.84 | 3143.30 | 3630.53 | 11.11 | 11.90 | 11.97 | 4950.01 | 140228/143771 (97.5%) | ⚠️ | +| c1_i131072_o256 | 131072 | 256 | tp4_dp2 | 1 | 0.61 | 61.94 | 2612.01 | 3033.01 | 7.81 | 8.23 | 8.24 | 3829.62 | 141374/143771 (98.3%) | ✅ | +| c1_i131072_o256 | 131072 | 256 | tp8_dp1 | 1 | 1.03 | 104.48 | 526.13 | 531.79 | 6.38 | 6.74 | 6.75 | 1365.14 | 134302/143771 (93.4%) | ✅ | +| c1_i131072_o512 | 131072 | 512 | tp2_dp4 | 1 | 0.67 | 68.11 | 574.50 | 577.45 | 10.89 | 11.54 | 11.54 | 2318.95 | 140228/143771 (97.5%) | ✅ | +| c1_i131072_o512 | 131072 | 512 | tp4_dp2 | 1 | 0.89 | 90.55 | 555.04 | 573.87 | 7.81 | 8.24 | 8.27 | 1649.69 | 141374/143771 (98.3%) | ✅ | +| c1_i131072_o512 | 131072 | 512 | tp8_dp1 | 1 | 1.03 | 104.09 | 528.58 | 529.52 | 6.38 | 6.74 | 6.75 | 1365.66 | 134302/143771 (93.4%) | ✅ | +| c1_i262144_o128 | 262144 | 128 | tp2_dp4 | 1 | 0.15 | 11.35 | 13444.27 | 14788.12 | 9.88 | 11.09 | 11.15 | 16032.30 | 140744/143771 (97.9%) | ⚠️ | +| c1_i262144_o128 | 262144 | 128 | tp4_dp2 | 1 | 0.20 | 15.35 | 8221.07 | 8613.48 | 6.70 | 7.64 | 7.66 | 9509.11 | 141374/143771 (98.3%) | ⚠️ | +| c1_i262144_o128 | 262144 | 128 | tp8_dp1 | 1 | 0.23 | 17.26 | 7547.16 | 7933.13 | 5.31 | 6.25 | 6.26 | 8667.71 | 134304/143771 (93.4%) | ⚠️ | +| c1_i262144_o256 | 262144 | 256 | tp2_dp4 | 1 | 0.18 | 18.19 | 8943.43 | 9385.72 | 10.07 | 11.43 | 11.50 | 10671.30 | 141258/143771 (98.3%) | ⚠️ | +| c1_i262144_o256 | 262144 | 256 | tp4_dp2 | 1 | 0.71 | 72.25 | 962.12 | 1036.06 | 6.92 | 7.88 | 7.93 | 2080.08 | 141374/143771 (98.3%) | ✅ | +| c1_i262144_o256 | 262144 | 256 | tp8_dp1 | 1 | 0.91 | 92.31 | 741.06 | 802.19 | 5.45 | 6.50 | 6.55 | 1664.47 | 134304/143771 (93.4%) | ✅ | +| c1_i524288_o128 | 524288 | 128 | tp2_dp4 | 1 | 0.13 | 9.76 | 17139.22 | 19091.80 | 9.70 | 10.39 | 10.40 | 20099.12 | 141258/143771 (98.3%) | ⚠️ | +| c1_i524288_o128 | 524288 | 128 | tp4_dp2 | 1 | 0.21 | 16.00 | 14806.51 | 17551.38 | 6.63 | 7.42 | 7.46 | 18281.61 | 141374/143771 (98.3%) | ⚠️ | +| c1_i524288_o128 | 524288 | 128 | tp8_dp1 | 1 | 0.23 | 17.65 | 13982.40 | 16622.77 | 5.18 | 6.10 | 6.17 | 17210.26 | 134304/143771 (93.4%) | ⚠️ | +| c1_i1048576_o128 | 1048576 | 128 | tp2_dp4 | 1 | 0.06 | 4.85 | 46081.66 | 51394.85 | 8.95 | 10.50 | 10.51 | 51849.45 | 141258/143771 (98.3%) | ⚠️ | +| c1_i1048576_o128 | 1048576 | 128 | tp4_dp2 | 1 | 0.06 | 4.88 | 43228.08 | 48170.56 | 6.14 | 7.51 | 7.56 | 48471.96 | 141374/143771 (98.3%) | ⚠️ | +| c1_i1048576_o128 | 1048576 | 128 | tp8_dp1 | 1 | 0.10 | 7.26 | 38265.96 | 45702.81 | 4.67 | 6.04 | 6.08 | 45912.47 | 134310/143771 (93.4%) | ⚠️ | +| c2_i262144_o128 | 262144 | 128 | tp2_dp4 | 2 | 0.27 | 20.65 | 9292.65 | 9433.23 | 24.30 | 74.40 | 75.57 | 10323.72 | 140744/143771 (97.9%) | ⚠️ | +| c2_i262144_o128 | 262144 | 128 | tp4_dp2 | 2 | 0.33 | 24.58 | 7994.94 | 8377.33 | 25.23 | 70.28 | 72.19 | 9106.67 | 141374/143771 (98.3%) | ⚠️ | +| c2_i262144_o128 | 262144 | 128 | tp8_dp1 | 2 | 0.58 | 43.70 | 6405.29 | 6864.92 | 12.04 | 40.84 | 63.02 | 7624.83 | 134304/143771 (93.4%) | ⚠️ | +| c2_i262144_o256 | 262144 | 256 | tp2_dp4 | 2 | 0.30 | 42.07 | 9153.02 | 10642.68 | 24.92 | 66.10 | 74.81 | 12104.02 | 141258/143771 (98.3%) | ⚠️ | +| c2_i262144_o256 | 262144 | 256 | tp4_dp2 | 2 | 0.48 | 66.52 | 6399.54 | 6424.61 | 10.75 | 22.96 | 32.38 | 8063.51 | 141374/143771 (98.3%) | ⚠️ | +| c2_i262144_o256 | 262144 | 256 | tp8_dp1 | 2 | 1.23 | 171.18 | 872.87 | 960.16 | 6.69 | 7.16 | 7.25 | 2191.39 | 134304/143771 (93.4%) | ✅ | +| c2_i524288_o128 | 524288 | 128 | tp2_dp4 | 2 | 0.69 | 52.24 | 4695.87 | 6463.46 | 15.77 | 44.14 | 65.40 | 8060.58 | 141258/143771 (98.3%) | ⚠️ | +| c2_i524288_o128 | 524288 | 128 | tp4_dp2 | 2 | 0.64 | 48.43 | 6241.67 | 9313.98 | 44.54 | 213.86 | 348.24 | 10806.98 | 141374/143771 (98.3%) | ⚠️ | +| c2_i524288_o128 | 524288 | 128 | tp8_dp1 | 2 | 1.76 | 133.35 | 1167.12 | 1232.75 | 5.64 | 6.90 | 6.94 | 1708.86 | 134306/143771 (93.4%) | ✅ | +| c2_i1048576_o128 | 1048576 | 128 | tp2_dp4 | 2 | 0.06 | 4.16 | 61414.70 | 65340.99 | 117.07 | 500.43 | 531.18 | 66294.91 | 141780/143771 (98.6%) | ❌ | +| c2_i1048576_o128 | 1048576 | 128 | tp4_dp2 | 2 | 0.06 | 4.56 | 51577.89 | 52987.83 | 277.94 | 766.41 | 858.05 | 56546.05 | 141374/143771 (98.3%) | ❌ | +| c2_i1048576_o128 | 1048576 | 128 | tp8_dp1 | 2 | 0.08 | 6.16 | 59936.57 | 60383.98 | 15.73 | 56.69 | 68.50 | 66073.78 | 134312/143771 (93.4%) | ⚠️ | +| c4_i131072_o128 | 131072 | 128 | tp2_dp4 | 4 | 0.74 | 41.97 | 5381.79 | 6485.26 | 63.46 | 234.23 | 254.36 | 9800.84 | 140228/143771 (97.5%) | ❌ | +| c4_i131072_o128 | 131072 | 128 | tp4_dp2 | 4 | 0.61 | 34.68 | 7688.92 | 7746.95 | 55.15 | 151.40 | 157.05 | 13546.32 | 141374/143771 (98.3%) | ❌ | +| c4_i131072_o128 | 131072 | 128 | tp8_dp1 | 4 | 0.82 | 46.76 | 5665.45 | 7563.45 | 46.92 | 130.55 | 135.39 | 9263.16 | 134302/143771 (93.4%) | ⚠️ | +| c4_i131072_o256 | 131072 | 256 | tp2_dp4 | 4 | 0.51 | 64.78 | 4431.12 | 6485.73 | 38.02 | 82.59 | 93.31 | 21326.10 | 140228/143771 (97.5%) | ⚠️ | +| c4_i131072_o256 | 131072 | 256 | tp4_dp2 | 4 | 0.79 | 101.20 | 6019.33 | 6092.09 | 20.62 | 43.90 | 46.90 | 10875.06 | 141374/143771 (98.3%) | ⚠️ | +| c4_i131072_o256 | 131072 | 256 | tp8_dp1 | 4 | 2.40 | 306.01 | 826.11 | 873.84 | 7.11 | 8.42 | 8.69 | 2602.99 | 134302/143771 (93.4%) | ✅ | +| c4_i131072_o512 | 131072 | 512 | tp2_dp4 | 4 | 0.46 | 136.39 | 5162.58 | 5751.13 | 24.26 | 43.94 | 51.73 | 14391.10 | 140228/143771 (97.5%) | ⚠️ | +| c4_i131072_o512 | 131072 | 512 | tp4_dp2 | 4 | 1.03 | 302.02 | 724.28 | 1360.67 | 9.94 | 13.97 | 16.16 | 5861.60 | 141374/143771 (98.3%) | ✅ | +| c4_i131072_o512 | 131072 | 512 | tp8_dp1 | 4 | 1.30 | 382.40 | 581.21 | 686.73 | 7.67 | 8.27 | 8.30 | 4365.16 | 134302/143771 (93.4%) | ✅ | +| c8_i65536_o128 | 65536 | 128 | tp2_dp4 | 8 | 0.86 | 55.23 | 5855.87 | 7061.54 | 115.39 | 327.74 | 399.92 | 18685.86 | 139708/143771 (97.2%) | ❌ | +| c8_i65536_o128 | 65536 | 128 | tp4_dp2 | 8 | 0.94 | 60.49 | 5620.08 | 5766.46 | 118.91 | 301.63 | 309.58 | 16549.56 | 140854/143771 (98.0%) | ❌ | +| c8_i65536_o128 | 65536 | 128 | tp8_dp1 | 8 | 1.01 | 64.77 | 7445.34 | 8645.53 | 88.42 | 248.75 | 263.80 | 16484.17 | 133780/143771 (93.1%) | ❌ | +| c8_i65536_o256 | 65536 | 256 | tp2_dp4 | 8 | 0.90 | 117.73 | 4029.43 | 4806.74 | 54.45 | 101.30 | 122.71 | 18052.15 | 139708/143771 (97.2%) | ❌ | +| c8_i65536_o256 | 65536 | 256 | tp4_dp2 | 8 | 1.31 | 171.79 | 3353.05 | 4860.71 | 44.63 | 113.59 | 141.58 | 15303.64 | 140854/143771 (98.0%) | ⚠️ | +| c8_i65536_o256 | 65536 | 256 | tp8_dp1 | 8 | 4.63 | 608.09 | 749.11 | 864.77 | 9.14 | 10.21 | 10.30 | 2998.37 | 133780/143771 (93.1%) | ✅ | +| c8_i65536_o512 | 65536 | 512 | tp2_dp4 | 8 | 0.98 | 272.84 | 3168.19 | 5671.91 | 24.56 | 51.52 | 85.56 | 18250.24 | 139708/143771 (97.2%) | ⚠️ | +| c8_i65536_o512 | 65536 | 512 | tp4_dp2 | 8 | 1.47 | 410.24 | 1809.22 | 2638.39 | 17.90 | 23.23 | 79.99 | 9676.38 | 140854/143771 (98.0%) | ✅ | +| c8_i65536_o512 | 65536 | 512 | tp8_dp1 | 8 | 2.53 | 705.69 | 661.39 | 679.12 | 9.10 | 9.63 | 10.39 | 5200.71 | 133780/143771 (93.1%) | ✅ | +| c8_i65536_o1024 | 65536 | 1024 | tp2_dp4 | 8 | 0.64 | 351.24 | 2850.09 | 2967.29 | 19.43 | 29.44 | 36.68 | 21507.37 | 139708/143771 (97.2%) | ✅ | +| c8_i65536_o1024 | 65536 | 1024 | tp4_dp2 | 8 | 0.97 | 528.38 | 2177.55 | 2783.15 | 13.13 | 20.42 | 32.65 | 13878.42 | 140854/143771 (98.0%) | ✅ | +| c8_i65536_o1024 | 65536 | 1024 | tp8_dp1 | 8 | 1.34 | 734.78 | 671.08 | 707.77 | 8.91 | 9.20 | 10.49 | 9476.99 | 133780/143771 (93.1%) | ✅ | +| c32_i16384_o128 | 16384 | 128 | tp2_dp4 | 32 | 4.31 | 284.72 | 3298.27 | 4051.80 | 107.21 | 186.59 | 376.91 | 15245.93 | 139324/143771 (96.9%) | ❌ | +| c32_i16384_o128 | 16384 | 128 | tp4_dp2 | 32 | 5.16 | 340.76 | 3479.51 | 3760.82 | 85.23 | 160.17 | 235.33 | 12999.09 | 139824/143771 (97.3%) | ❌ | +| c32_i16384_o128 | 16384 | 128 | tp8_dp1 | 32 | 5.39 | 356.17 | 3789.36 | 4532.55 | 78.86 | 150.73 | 226.61 | 11830.35 | 132932/143771 (92.5%) | ❌ | +| c32_i16384_o256 | 16384 | 256 | tp2_dp4 | 32 | 4.49 | 623.03 | 1913.91 | 2384.97 | 47.34 | 71.67 | 147.97 | 13936.33 | 139684/143771 (97.2%) | ✅ | +| c32_i16384_o256 | 16384 | 256 | tp4_dp2 | 32 | 5.41 | 751.48 | 1701.48 | 2572.97 | 38.88 | 55.35 | 126.86 | 11645.26 | 140338/143771 (97.6%) | ✅ | +| c32_i16384_o256 | 16384 | 256 | tp8_dp1 | 32 | 12.84 | 1782.54 | 636.48 | 644.96 | 14.84 | 16.94 | 20.78 | 4262.99 | 132932/143771 (92.5%) | ✅ | +| c32_i16384_o512 | 16384 | 512 | tp2_dp4 | 32 | 3.77 | 952.48 | 1520.07 | 2259.12 | 30.21 | 45.07 | 57.04 | 18769.12 | 139684/143771 (97.2%) | ✅ | +| c32_i16384_o512 | 16384 | 512 | tp4_dp2 | 32 | 5.44 | 1373.00 | 965.45 | 1010.01 | 20.31 | 25.61 | 28.12 | 12002.66 | 140854/143771 (98.0%) | ✅ | +| c32_i16384_o512 | 16384 | 512 | tp8_dp1 | 32 | 7.83 | 1977.19 | 640.86 | 739.06 | 14.04 | 15.70 | 18.53 | 7504.82 | 132932/143771 (92.5%) | ✅ | +| c32_i16384_o1024 | 16384 | 1024 | tp2_dp4 | 32 | 2.52 | 1337.54 | 1871.97 | 2404.37 | 22.95 | 27.86 | 72.28 | 22977.81 | 139684/143771 (97.2%) | ✅ | +| c32_i16384_o1024 | 16384 | 1024 | tp4_dp2 | 32 | 3.21 | 1704.78 | 1131.20 | 1660.57 | 17.14 | 21.79 | 37.19 | 18824.49 | 140854/143771 (98.0%) | ✅ | +| c32_i16384_o1024 | 16384 | 1024 | tp8_dp1 | 32 | 4.00 | 2122.35 | 789.94 | 950.40 | 13.38 | 14.23 | 22.02 | 13797.93 | 132932/143771 (92.5%) | ✅ | +| c32_i16384_o2048 | 16384 | 2048 | tp2_dp4 | 32 | 1.56 | 1785.47 | 1089.13 | 1111.26 | 15.83 | 17.47 | 19.70 | 33430.94 | 139684/143771 (97.2%) | ✅ | +| c32_i16384_o2048 | 16384 | 2048 | tp4_dp2 | 32 | 1.73 | 1982.72 | 1143.92 | 2172.49 | 14.60 | 16.68 | 25.49 | 30322.94 | 140854/143771 (98.0%) | ✅ | +| c32_i16384_o2048 | 16384 | 2048 | tp8_dp1 | 32 | 1.94 | 2226.25 | 628.02 | 806.24 | 12.88 | 13.39 | 14.85 | 26578.49 | 132932/143771 (92.5%) | ✅ | +| c64_i4096_o128 | 4096 | 128 | tp2_dp4 | 64 | 11.02 | 721.10 | 1790.01 | 2034.19 | 83.97 | 111.62 | 188.61 | 12237.14 | 139324/143771 (96.9%) | ⚠️ | +| c64_i4096_o128 | 4096 | 128 | tp4_dp2 | 64 | 10.43 | 682.69 | 1904.20 | 2262.19 | 88.50 | 123.52 | 173.68 | 12463.74 | 139456/143771 (97.0%) | ⚠️ | +| c64_i4096_o128 | 4096 | 128 | tp8_dp1 | 64 | 11.81 | 772.92 | 2978.27 | 3725.28 | 76.32 | 108.07 | 233.29 | 11989.51 | 132932/143771 (92.5%) | ⚠️ | +| c64_i4096_o256 | 4096 | 256 | tp2_dp4 | 64 | 8.50 | 1101.01 | 1290.52 | 1667.29 | 55.00 | 72.09 | 89.93 | 15631.13 | 139324/143771 (96.9%) | ⚠️ | +| c64_i4096_o256 | 4096 | 256 | tp4_dp2 | 64 | 10.23 | 1323.77 | 1198.05 | 1234.72 | 45.94 | 59.77 | 82.58 | 13085.09 | 139456/143771 (97.0%) | ✅ | +| c64_i4096_o256 | 4096 | 256 | tp8_dp1 | 64 | 19.45 | 2518.43 | 804.28 | 815.33 | 22.25 | 25.88 | 28.53 | 6644.63 | 132932/143771 (92.5%) | ✅ | +| c64_i4096_o512 | 4096 | 512 | tp2_dp4 | 64 | 6.47 | 1712.46 | 1099.58 | 1280.63 | 32.92 | 40.95 | 51.15 | 18838.52 | 139324/143771 (96.9%) | ✅ | +| c64_i4096_o512 | 4096 | 512 | tp4_dp2 | 64 | 8.10 | 2143.08 | 946.13 | 974.08 | 26.25 | 32.55 | 35.60 | 15173.06 | 139456/143771 (97.0%) | ✅ | +| c64_i4096_o512 | 4096 | 512 | tp8_dp1 | 64 | 11.71 | 3100.03 | 584.91 | 601.95 | 17.83 | 18.78 | 19.95 | 9668.74 | 132932/143771 (92.5%) | ✅ | +| c64_i4096_o1024 | 4096 | 1024 | tp2_dp4 | 64 | 4.39 | 2351.38 | 1083.22 | 1200.04 | 24.41 | 28.13 | 32.24 | 26088.66 | 139324/143771 (96.9%) | ✅ | +| c64_i4096_o1024 | 4096 | 1024 | tp4_dp2 | 64 | 5.45 | 2915.21 | 1023.51 | 1212.54 | 19.18 | 21.78 | 24.49 | 21920.34 | 139824/143771 (97.3%) | ✅ | +| c64_i4096_o1024 | 4096 | 1024 | tp8_dp1 | 64 | 6.14 | 3283.93 | 789.79 | 800.08 | 17.06 | 17.95 | 19.33 | 17906.82 | 132932/143771 (92.5%) | ✅ | +| c64_i4096_o2048 | 4096 | 2048 | tp2_dp4 | 64 | 2.73 | 2846.07 | 1149.71 | 1194.86 | 19.80 | 22.43 | 24.96 | 41227.42 | 139324/143771 (96.9%) | ✅ | +| c64_i4096_o2048 | 4096 | 2048 | tp4_dp2 | 64 | 3.04 | 3165.95 | 856.07 | 869.36 | 17.64 | 18.77 | 20.69 | 37188.62 | 139824/143771 (97.3%) | ✅ | +| c64_i4096_o2048 | 4096 | 2048 | tp8_dp1 | 64 | 3.28 | 3408.88 | 570.75 | 582.03 | 16.56 | 17.06 | 18.07 | 34336.54 | 132932/143771 (92.5%) | ✅ | +| c64_i4096_o4096 | 4096 | 4096 | tp2_dp4 | 64 | 1.49 | 3050.95 | 900.93 | 999.74 | 17.93 | 19.28 | 20.04 | 75036.72 | 139324/143771 (96.9%) | ✅ | +| c64_i4096_o4096 | 4096 | 4096 | tp4_dp2 | 64 | 1.62 | 3323.06 | 727.11 | 753.66 | 16.53 | 17.22 | 17.64 | 68039.71 | 139824/143771 (97.3%) | ✅ | +| c64_i4096_o4096 | 4096 | 4096 | tp8_dp1 | 64 | 1.67 | 3408.94 | 576.41 | 599.00 | 16.44 | 16.79 | 16.98 | 68183.67 | 132932/143771 (92.5%) | ✅ | +| c128_i1024_o128 | 1024 | 128 | tp2_dp4 | 128 | 14.09 | 919.53 | 1787.94 | 1815.68 | 131.99 | 159.76 | 191.60 | 19385.13 | 139322/143771 (96.9%) | ⚠️ | +| c128_i1024_o128 | 1024 | 128 | tp4_dp2 | 128 | 15.24 | 994.43 | 1659.18 | 1917.69 | 122.83 | 156.63 | 189.20 | 18386.47 | 139454/143771 (97.0%) | ⚠️ | +| c128_i1024_o128 | 1024 | 128 | tp8_dp1 | 128 | 25.21 | 1644.87 | 1647.93 | 2127.36 | 71.51 | 108.98 | 146.42 | 11987.25 | 132930/143771 (92.5%) | ⚠️ | +| c128_i1024_o256 | 1024 | 256 | tp2_dp4 | 128 | 12.51 | 1654.84 | 1302.99 | 1324.67 | 72.57 | 92.16 | 105.85 | 20487.34 | 139322/143771 (96.9%) | ⚠️ | +| c128_i1024_o256 | 1024 | 256 | tp4_dp2 | 128 | 15.10 | 1996.42 | 938.43 | 1171.74 | 60.05 | 74.97 | 96.47 | 16952.04 | 139454/143771 (97.0%) | ⚠️ | +| c128_i1024_o256 | 1024 | 256 | tp8_dp1 | 128 | 23.31 | 3082.56 | 725.46 | 741.75 | 37.61 | 47.17 | 56.15 | 10662.58 | 132930/143771 (92.5%) | ✅ | +| c128_i1024_o512 | 1024 | 512 | tp2_dp4 | 128 | 9.80 | 2483.34 | 1236.35 | 1486.37 | 47.72 | 59.20 | 70.21 | 26017.94 | 139322/143771 (96.9%) | ✅ | +| c128_i1024_o512 | 1024 | 512 | tp4_dp2 | 128 | 11.78 | 2986.46 | 996.03 | 1011.18 | 39.13 | 48.34 | 56.02 | 22283.80 | 139456/143771 (97.0%) | ✅ | +| c128_i1024_o512 | 1024 | 512 | tp8_dp1 | 128 | 17.09 | 4330.93 | 760.28 | 777.02 | 26.29 | 29.37 | 34.23 | 14073.81 | 132932/143771 (92.5%) | ✅ | +| c128_i1024_o1024 | 1024 | 1024 | tp2_dp4 | 128 | 7.08 | 3578.74 | 1147.11 | 1259.64 | 32.79 | 39.48 | 50.72 | 34693.91 | 139322/143771 (96.9%) | ✅ | +| c128_i1024_o1024 | 1024 | 1024 | tp4_dp2 | 128 | 8.62 | 4359.31 | 719.08 | 837.92 | 26.31 | 30.84 | 33.28 | 27367.26 | 139456/143771 (97.0%) | ✅ | +| c128_i1024_o1024 | 1024 | 1024 | tp8_dp1 | 128 | 9.35 | 4727.69 | 724.06 | 738.02 | 24.27 | 25.99 | 28.67 | 25355.91 | 132932/143771 (92.5%) | ✅ | +| c128_i1024_o2048 | 1024 | 2048 | tp2_dp4 | 128 | 4.40 | 4467.90 | 990.93 | 1038.67 | 25.24 | 28.25 | 31.35 | 52638.14 | 139322/143771 (96.9%) | ✅ | +| c128_i1024_o2048 | 1024 | 2048 | tp4_dp2 | 128 | 4.83 | 4909.69 | 979.55 | 1002.42 | 22.76 | 24.25 | 25.28 | 47303.18 | 139456/143771 (97.0%) | ✅ | +| c128_i1024_o2048 | 1024 | 2048 | tp8_dp1 | 128 | 4.80 | 4871.12 | 746.05 | 761.60 | 23.37 | 24.26 | 24.89 | 48201.59 | 132932/143771 (92.5%) | ✅ | +| c128_i1024_o4096 | 1024 | 4096 | tp2_dp4 | 128 | 2.42 | 4961.16 | 901.88 | 946.01 | 22.33 | 23.78 | 25.83 | 91949.11 | 139322/143771 (96.9%) | ✅ | +| c128_i1024_o4096 | 1024 | 4096 | tp4_dp2 | 128 | 2.49 | 5110.60 | 737.03 | 754.71 | 21.93 | 23.01 | 23.26 | 90427.39 | 139456/143771 (97.0%) | ✅ | +| c128_i1024_o4096 | 1024 | 4096 | tp8_dp1 | 128 | 2.33 | 4775.68 | 736.09 | 756.63 | 23.99 | 25.92 | 26.62 | 97316.79 | 132932/143771 (92.5%) | ✅ | + +## Best throughput per (ISL, DSL) + +| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO | +|---:|---:|---|---:|---:|---:|---:|---:| +| 1024 | 128 | tp8_dp1 | 128 | 1644.87 | 1647.93 | 71.51 | ⚠️ | +| 1024 | 256 | tp8_dp1 | 128 | 3082.56 | 725.46 | 37.61 | ✅ | +| 1024 | 512 | tp8_dp1 | 128 | 4330.93 | 760.28 | 26.29 | ✅ | +| 1024 | 1024 | tp8_dp1 | 128 | 4727.69 | 724.06 | 24.27 | ✅ | +| 1024 | 2048 | tp4_dp2 | 128 | 4909.69 | 979.55 | 22.76 | ✅ | +| 1024 | 4096 | tp4_dp2 | 128 | 5110.60 | 737.03 | 21.93 | ✅ | +| 4096 | 128 | tp8_dp1 | 64 | 772.92 | 2978.27 | 76.32 | ⚠️ | +| 4096 | 256 | tp8_dp1 | 64 | 2518.43 | 804.28 | 22.25 | ✅ | +| 4096 | 512 | tp8_dp1 | 64 | 3100.03 | 584.91 | 17.83 | ✅ | +| 4096 | 1024 | tp8_dp1 | 64 | 3283.93 | 789.79 | 17.06 | ✅ | +| 4096 | 2048 | tp8_dp1 | 64 | 3408.88 | 570.75 | 16.56 | ✅ | +| 4096 | 4096 | tp8_dp1 | 64 | 3408.94 | 576.41 | 16.44 | ✅ | +| 16384 | 128 | tp8_dp1 | 32 | 356.17 | 3789.36 | 78.86 | ❌ | +| 16384 | 256 | tp8_dp1 | 32 | 1782.54 | 636.48 | 14.84 | ✅ | +| 16384 | 512 | tp8_dp1 | 32 | 1977.19 | 640.86 | 14.04 | ✅ | +| 16384 | 1024 | tp8_dp1 | 32 | 2122.35 | 789.94 | 13.38 | ✅ | +| 16384 | 2048 | tp8_dp1 | 32 | 2226.25 | 628.02 | 12.88 | ✅ | +| 65536 | 128 | tp8_dp1 | 8 | 64.77 | 7445.34 | 88.42 | ❌ | +| 65536 | 256 | tp8_dp1 | 8 | 608.09 | 749.11 | 9.14 | ✅ | +| 65536 | 512 | tp8_dp1 | 8 | 705.69 | 661.39 | 9.10 | ✅ | +| 65536 | 1024 | tp8_dp1 | 8 | 734.78 | 671.08 | 8.91 | ✅ | +| 131072 | 128 | tp8_dp1 | 4 | 46.76 | 5665.45 | 46.92 | ⚠️ | +| 131072 | 256 | tp8_dp1 | 4 | 306.01 | 826.11 | 7.11 | ✅ | +| 131072 | 512 | tp8_dp1 | 4 | 382.40 | 581.21 | 7.67 | ✅ | +| 262144 | 128 | tp8_dp1 | 2 | 43.70 | 6405.29 | 12.04 | ⚠️ | +| 262144 | 256 | tp8_dp1 | 2 | 171.18 | 872.87 | 6.69 | ✅ | +| 524288 | 128 | tp8_dp1 | 2 | 133.35 | 1167.12 | 5.64 | ✅ | +| 1048576 | 128 | tp8_dp1 | 1 | 7.26 | 38265.96 | 4.67 | ⚠️ | + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out. +- `Peak GPU mem` shows peak used / total MB and utilization percentage. +- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run. diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/skipped_after_oom.csv b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/skipped_after_oom.csv new file mode 100644 index 0000000..46d89ab --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/skipped_after_oom.csv @@ -0,0 +1 @@ +engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/report.md b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/report.md new file mode 100644 index 0000000..340263b --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/report.md @@ -0,0 +1,70 @@ +# 8x NVIDIA H200 143GB VLLM Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: VLLM +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i1024_o128 | main | 1 | 1024 | 128 | 4.76 | 5 | 1.05 | 373.68 | 79.65 | 453.33 | 159.97 | 319.08 | 355.57 | 10.56 | 10.99 | 11.00 | 950.14 | 1366.62 | 1406.84 | 138264/143771 MiB (96.2%) | ✅ | +| c128_i1024_o128 | main | 128 | 1024 | 128 | 45.42 | 640 | 14.09 | 7372.62 | 919.53 | 8292.15 | 682.86 | 1787.94 | 1815.68 | 131.99 | 159.76 | 191.60 | 8857.59 | 17800.52 | 19385.13 | 139322/143771 MiB (96.9%) | ⚠️ | +| c1_i1024_o256 | main | 1 | 1024 | 256 | 5.98 | 5 | 0.84 | 297.55 | 84.85 | 382.39 | 96.69 | 107.93 | 108.38 | 11.01 | 11.57 | 11.61 | 1193.49 | 1982.06 | 2082.56 | 139322/143771 MiB (96.9%) | ✅ | +| c128_i1024_o256 | main | 128 | 1024 | 256 | 51.15 | 640 | 12.51 | 6546.63 | 1654.84 | 8201.47 | 455.67 | 1302.99 | 1324.67 | 72.57 | 92.16 | 105.85 | 9769.30 | 18928.98 | 20487.34 | 139322/143771 MiB (96.9%) | ⚠️ | +| c1_i1024_o512 | main | 1 | 1024 | 512 | 6.34 | 5 | 0.79 | 280.27 | 79.92 | 360.18 | 95.33 | 113.71 | 115.02 | 11.50 | 11.85 | 11.86 | 1267.01 | 2173.11 | 2303.60 | 139322/143771 MiB (96.9%) | ✅ | +| c128_i1024_o512 | main | 128 | 1024 | 512 | 65.32 | 640 | 9.80 | 5126.38 | 2483.34 | 7609.72 | 392.20 | 1236.35 | 1486.37 | 47.72 | 59.20 | 70.21 | 12165.11 | 24348.08 | 26017.94 | 139322/143771 MiB (96.9%) | ✅ | +| c1_i1024_o1024 | main | 1 | 1024 | 1024 | 18.32 | 5 | 0.27 | 97.06 | 83.58 | 180.64 | 98.32 | 100.14 | 100.52 | 11.71 | 11.90 | 11.93 | 3661.85 | 8044.41 | 8200.76 | 139322/143771 MiB (96.9%) | ✅ | +| c128_i1024_o1024 | main | 128 | 1024 | 1024 | 90.39 | 640 | 7.08 | 3704.45 | 3578.74 | 7283.20 | 340.83 | 1147.11 | 1259.64 | 32.79 | 39.48 | 50.72 | 16393.11 | 32588.31 | 34693.91 | 139322/143771 MiB (96.9%) | ✅ | +| c1_i1024_o2048 | main | 1 | 1024 | 2048 | 66.40 | 5 | 0.08 | 26.78 | 84.74 | 111.51 | 95.99 | 105.11 | 105.45 | 11.70 | 11.81 | 11.81 | 13278.26 | 20062.55 | 20192.89 | 139322/143771 MiB (96.9%) | ✅ | +| c128_i1024_o2048 | main | 128 | 1024 | 2048 | 145.52 | 640 | 4.40 | 2301.16 | 4467.90 | 6769.06 | 282.02 | 990.93 | 1038.67 | 25.24 | 28.25 | 31.35 | 25564.98 | 48968.45 | 52638.14 | 139322/143771 MiB (96.9%) | ✅ | +| c1_i1024_o4096 | main | 1 | 1024 | 4096 | 136.81 | 5 | 0.04 | 13.00 | 86.04 | 99.04 | 98.28 | 114.24 | 116.37 | 11.59 | 11.84 | 11.86 | 27359.51 | 42290.37 | 43455.08 | 139322/143771 MiB (96.9%) | ✅ | +| c128_i1024_o4096 | main | 128 | 1024 | 4096 | 264.80 | 640 | 2.42 | 1264.57 | 4961.16 | 6225.74 | 265.10 | 901.88 | 946.01 | 22.33 | 23.78 | 25.83 | 45632.76 | 86442.33 | 91949.11 | 139322/143771 MiB (96.9%) | ✅ | +| c1_i4096_o128 | main | 1 | 4096 | 128 | 5.45 | 5 | 0.92 | 1829.89 | 69.56 | 1899.45 | 221.18 | 353.45 | 371.20 | 11.63 | 11.95 | 12.00 | 1088.26 | 1560.46 | 1608.68 | 139322/143771 MiB (96.9%) | ✅ | +| c64_i4096_o128 | main | 64 | 4096 | 128 | 29.05 | 320 | 11.02 | 23259.85 | 721.10 | 23980.95 | 566.58 | 1790.01 | 2034.19 | 83.97 | 111.62 | 188.61 | 5607.28 | 11212.93 | 12237.14 | 139324/143771 MiB (96.9%) | ⚠️ | +| c1_i4096_o256 | main | 1 | 4096 | 256 | 6.50 | 5 | 0.77 | 1534.00 | 78.01 | 1612.01 | 128.46 | 208.90 | 226.63 | 11.61 | 11.87 | 11.89 | 1298.45 | 2189.81 | 2296.91 | 139324/143771 MiB (96.9%) | ✅ | +| c64_i4096_o256 | main | 64 | 4096 | 256 | 37.63 | 320 | 8.50 | 17956.97 | 1101.01 | 19057.98 | 447.33 | 1290.52 | 1667.29 | 55.00 | 72.09 | 89.93 | 7132.99 | 14073.09 | 15631.13 | 139324/143771 MiB (96.9%) | ⚠️ | +| c1_i4096_o512 | main | 1 | 4096 | 512 | 6.48 | 5 | 0.77 | 1539.75 | 78.30 | 1618.05 | 124.97 | 210.04 | 232.13 | 11.63 | 11.95 | 11.98 | 1293.49 | 2240.30 | 2378.47 | 139324/143771 MiB (96.9%) | ✅ | +| c64_i4096_o512 | main | 64 | 4096 | 512 | 49.46 | 320 | 6.47 | 13661.57 | 1712.46 | 15374.03 | 363.21 | 1099.58 | 1280.63 | 32.92 | 40.95 | 51.15 | 9074.10 | 17834.26 | 18838.52 | 139324/143771 MiB (96.9%) | ✅ | +| c1_i4096_o1024 | main | 1 | 4096 | 1024 | 18.31 | 5 | 0.27 | 544.41 | 83.60 | 628.01 | 103.66 | 113.91 | 115.64 | 11.55 | 11.86 | 11.87 | 3661.10 | 8164.68 | 8360.74 | 139324/143771 MiB (96.9%) | ✅ | +| c64_i4096_o1024 | main | 64 | 4096 | 1024 | 72.82 | 320 | 4.39 | 9278.68 | 2351.38 | 11630.06 | 336.61 | 1083.22 | 1200.04 | 24.41 | 28.13 | 32.24 | 13050.59 | 25157.60 | 26088.66 | 139324/143771 MiB (96.9%) | ✅ | +| c1_i4096_o2048 | main | 1 | 4096 | 2048 | 65.21 | 5 | 0.08 | 152.90 | 86.30 | 239.20 | 107.97 | 122.31 | 124.03 | 11.54 | 11.75 | 11.78 | 13039.42 | 19438.63 | 19528.19 | 139324/143771 MiB (96.9%) | ✅ | +| c64_i4096_o2048 | main | 64 | 4096 | 2048 | 117.01 | 320 | 2.73 | 5774.33 | 2846.07 | 8620.40 | 297.64 | 1149.71 | 1194.86 | 19.80 | 22.43 | 24.96 | 20535.82 | 39745.31 | 41227.42 | 139324/143771 MiB (96.9%) | ✅ | +| c1_i4096_o4096 | main | 1 | 4096 | 4096 | 137.51 | 5 | 0.04 | 72.50 | 85.60 | 158.10 | 114.72 | 130.43 | 132.68 | 11.66 | 11.87 | 11.89 | 27500.53 | 42270.91 | 43588.26 | 139324/143771 MiB (96.9%) | ✅ | +| c64_i4096_o4096 | main | 64 | 4096 | 4096 | 214.54 | 320 | 1.49 | 3149.26 | 3050.95 | 6200.21 | 274.73 | 900.93 | 999.74 | 17.93 | 19.28 | 20.04 | 36687.19 | 72324.60 | 75036.72 | 139324/143771 MiB (96.9%) | ✅ | +| c1_i16384_o128 | main | 1 | 16384 | 128 | 5.71 | 5 | 0.88 | 7481.41 | 66.35 | 7547.76 | 320.96 | 599.10 | 618.22 | 11.00 | 11.48 | 11.54 | 1140.76 | 1856.53 | 1948.72 | 139324/143771 MiB (96.9%) | ✅ | +| c32_i16384_o128 | main | 32 | 16384 | 128 | 37.11 | 160 | 4.31 | 34791.25 | 284.72 | 35075.97 | 956.28 | 3298.27 | 4051.80 | 107.21 | 186.59 | 376.91 | 7222.82 | 13755.15 | 15245.93 | 139324/143771 MiB (96.9%) | ❌ | +| c1_i16384_o256 | main | 1 | 16384 | 256 | 7.07 | 5 | 0.71 | 6043.42 | 71.69 | 6115.12 | 256.48 | 448.96 | 486.54 | 11.52 | 11.66 | 11.67 | 1412.57 | 2447.12 | 2621.34 | 139324/143771 MiB (96.9%) | ✅ | +| c32_i16384_o256 | main | 32 | 16384 | 256 | 35.65 | 160 | 4.49 | 36211.60 | 623.03 | 36834.63 | 680.08 | 1913.91 | 2384.97 | 47.34 | 71.67 | 147.97 | 6749.38 | 12780.12 | 13936.33 | 139684/143771 MiB (97.2%) | ✅ | +| c1_i16384_o512 | main | 1 | 16384 | 512 | 6.52 | 5 | 0.77 | 6550.19 | 77.70 | 6627.89 | 139.05 | 188.54 | 190.96 | 11.55 | 11.73 | 11.74 | 1303.13 | 2227.30 | 2356.50 | 139684/143771 MiB (97.2%) | ✅ | +| c32_i16384_o512 | main | 32 | 16384 | 512 | 42.41 | 160 | 3.77 | 30447.27 | 952.48 | 31399.75 | 538.18 | 1520.07 | 2259.12 | 30.21 | 45.07 | 57.04 | 7937.47 | 16413.95 | 18769.12 | 139684/143771 MiB (97.2%) | ✅ | +| c1_i16384_o1024 | main | 1 | 16384 | 1024 | 18.25 | 5 | 0.27 | 2341.54 | 83.88 | 2425.42 | 188.47 | 322.92 | 349.35 | 11.31 | 11.64 | 11.64 | 3648.93 | 7809.61 | 7915.88 | 139684/143771 MiB (97.2%) | ✅ | +| c32_i16384_o1024 | main | 32 | 16384 | 1024 | 63.50 | 160 | 2.52 | 20332.49 | 1337.54 | 21670.03 | 485.63 | 1871.97 | 2404.37 | 22.95 | 27.86 | 72.28 | 11657.91 | 21919.16 | 22977.81 | 139684/143771 MiB (97.2%) | ✅ | +| c1_i16384_o2048 | main | 1 | 16384 | 2048 | 65.72 | 5 | 0.08 | 650.30 | 85.62 | 735.91 | 175.36 | 249.46 | 261.08 | 11.54 | 11.64 | 11.65 | 13142.27 | 19985.51 | 20210.50 | 139684/143771 MiB (97.2%) | ✅ | +| c32_i16384_o2048 | main | 32 | 16384 | 2048 | 102.63 | 160 | 1.56 | 12580.64 | 1785.47 | 14366.11 | 336.45 | 1089.13 | 1111.26 | 15.83 | 17.47 | 19.70 | 18450.05 | 31359.53 | 33430.94 | 139684/143771 MiB (97.2%) | ✅ | +| c1_i65536_o128 | main | 1 | 65536 | 128 | 11.40 | 5 | 0.44 | 15249.82 | 33.25 | 15283.07 | 1429.50 | 2811.65 | 2892.88 | 11.47 | 12.08 | 12.21 | 2278.14 | 4059.40 | 4233.91 | 139706/143771 MiB (97.2%) | ✅ | +| c8_i65536_o128 | main | 8 | 65536 | 128 | 46.50 | 40 | 0.86 | 31649.77 | 55.23 | 31705.00 | 2642.31 | 5855.87 | 7061.54 | 115.39 | 327.74 | 399.92 | 9193.45 | 17992.56 | 18685.86 | 139708/143771 MiB (97.2%) | ❌ | +| c1_i65536_o256 | main | 1 | 65536 | 256 | 10.35 | 5 | 0.48 | 16785.56 | 48.96 | 16834.52 | 902.47 | 2181.49 | 2314.91 | 11.55 | 11.95 | 11.96 | 2069.19 | 3540.82 | 3709.96 | 139708/143771 MiB (97.2%) | ✅ | +| c8_i65536_o256 | main | 8 | 65536 | 256 | 44.64 | 40 | 0.90 | 32962.95 | 117.73 | 33080.69 | 1800.38 | 4029.43 | 4806.74 | 54.45 | 101.30 | 122.71 | 8621.68 | 16550.72 | 18052.15 | 139708/143771 MiB (97.2%) | ❌ | +| c1_i65536_o512 | main | 1 | 65536 | 512 | 8.02 | 5 | 0.62 | 21662.85 | 63.19 | 21726.04 | 501.88 | 1276.37 | 1453.96 | 11.06 | 11.68 | 11.78 | 1602.98 | 2564.65 | 2624.42 | 139708/143771 MiB (97.2%) | ✅ | +| c8_i65536_o512 | main | 8 | 65536 | 512 | 40.84 | 40 | 0.98 | 36028.75 | 272.84 | 36301.59 | 1260.91 | 3168.19 | 5671.91 | 24.56 | 51.52 | 85.56 | 7629.47 | 14481.52 | 18250.24 | 139708/143771 MiB (97.2%) | ⚠️ | +| c1_i65536_o1024 | main | 1 | 65536 | 1024 | 18.42 | 5 | 0.27 | 9437.05 | 83.13 | 9520.17 | 255.57 | 375.58 | 383.90 | 11.12 | 11.57 | 11.58 | 3681.79 | 8015.79 | 8213.89 | 139708/143771 MiB (97.2%) | ✅ | +| c8_i65536_o1024 | main | 8 | 65536 | 1024 | 62.34 | 40 | 0.64 | 23606.18 | 351.24 | 23957.42 | 962.35 | 2850.09 | 2967.29 | 19.43 | 29.44 | 36.68 | 11157.95 | 21105.82 | 21507.37 | 139708/143771 MiB (97.2%) | ✅ | +| c1_i131072_o128 | main | 1 | 131072 | 128 | 15.53 | 5 | 0.32 | 23847.78 | 24.40 | 23872.18 | 2286.42 | 3831.71 | 3847.18 | 11.06 | 11.65 | 11.68 | 3104.64 | 4990.35 | 5071.55 | 140228/143771 MiB (97.5%) | ⚠️ | +| c4_i131072_o128 | main | 4 | 131072 | 128 | 27.14 | 20 | 0.74 | 54328.24 | 41.97 | 54370.21 | 2648.47 | 5381.79 | 6485.26 | 63.46 | 234.23 | 254.36 | 5273.33 | 9530.16 | 9800.84 | 140228/143771 MiB (97.5%) | ❌ | +| c1_i131072_o256 | main | 1 | 131072 | 256 | 10.82 | 5 | 0.46 | 34222.74 | 46.84 | 34269.59 | 1053.51 | 3143.30 | 3630.53 | 11.11 | 11.90 | 11.97 | 2162.71 | 4514.33 | 4950.01 | 140228/143771 MiB (97.5%) | ⚠️ | +| c4_i131072_o256 | main | 4 | 131072 | 256 | 39.32 | 20 | 0.51 | 37498.25 | 64.78 | 37563.03 | 2168.98 | 4431.12 | 6485.73 | 38.02 | 82.59 | 93.31 | 7621.59 | 19293.47 | 21326.10 | 140228/143771 MiB (97.5%) | ⚠️ | +| c1_i131072_o512 | main | 1 | 131072 | 512 | 7.44 | 5 | 0.67 | 49763.22 | 68.11 | 49831.33 | 391.27 | 574.50 | 577.45 | 10.89 | 11.54 | 11.54 | 1487.11 | 2243.60 | 2318.95 | 140228/143771 MiB (97.5%) | ✅ | +| c4_i131072_o512 | main | 4 | 131072 | 512 | 43.08 | 20 | 0.46 | 34227.86 | 136.39 | 34364.24 | 1588.49 | 5162.58 | 5751.13 | 24.26 | 43.94 | 51.73 | 8028.29 | 13969.64 | 14391.10 | 140228/143771 MiB (97.5%) | ⚠️ | +| c1_i262144_o128 | main | 1 | 262144 | 128 | 33.40 | 5 | 0.15 | 22865.42 | 11.35 | 22876.77 | 5896.15 | 13444.27 | 14788.12 | 9.88 | 11.09 | 11.15 | 6677.66 | 14582.28 | 16032.30 | 140744/143771 MiB (97.9%) | ⚠️ | +| c2_i262144_o128 | main | 2 | 262144 | 128 | 36.61 | 10 | 0.27 | 38069.90 | 20.65 | 38090.55 | 4480.80 | 9292.65 | 9433.23 | 24.30 | 74.40 | 75.57 | 6665.65 | 10320.34 | 10323.72 | 140744/143771 MiB (97.9%) | ⚠️ | +| c1_i262144_o256 | main | 1 | 262144 | 256 | 27.87 | 5 | 0.18 | 27404.52 | 18.19 | 27422.71 | 4494.81 | 8943.43 | 9385.72 | 10.07 | 11.43 | 11.50 | 5571.33 | 10356.69 | 10671.30 | 141258/143771 MiB (98.3%) | ⚠️ | +| c2_i262144_o256 | main | 2 | 262144 | 256 | 33.18 | 10 | 0.30 | 42010.29 | 42.07 | 42052.36 | 2785.85 | 9153.02 | 10642.68 | 24.92 | 66.10 | 74.81 | 6437.85 | 12049.75 | 12104.02 | 141258/143771 MiB (98.3%) | ⚠️ | +| c1_i524288_o128 | main | 1 | 524288 | 128 | 38.85 | 5 | 0.13 | 26404.42 | 9.76 | 26414.18 | 7011.27 | 17139.22 | 19091.80 | 9.70 | 10.39 | 10.40 | 7768.26 | 18082.00 | 20099.12 | 141258/143771 MiB (98.3%) | ⚠️ | +| c2_i524288_o128 | main | 2 | 524288 | 128 | 14.47 | 10 | 0.69 | 114432.83 | 52.24 | 114485.07 | 1445.03 | 4695.87 | 6463.46 | 15.77 | 44.14 | 65.40 | 2876.67 | 7604.53 | 8060.58 | 141258/143771 MiB (98.3%) | ⚠️ | +| c1_i1048576_o128 | main | 1 | 1048576 | 128 | 78.14 | 5 | 0.06 | 19836.89 | 4.85 | 19841.74 | 14906.36 | 46081.66 | 51394.85 | 8.95 | 10.50 | 10.51 | 15626.56 | 46632.30 | 51849.45 | 141258/143771 MiB (98.3%) | ⚠️ | +| c2_i1048576_o128 | main | 2 | 1048576 | 128 | 181.75 | 10 | 0.06 | 17765.33 | 4.16 | 17769.49 | 24258.31 | 61414.70 | 65340.99 | 117.07 | 500.43 | 531.18 | 30965.88 | 62664.75 | 66294.91 | 141780/143771 MiB (98.6%) | ❌ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/results.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/results.json new file mode 100644 index 0000000..114a410 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/results.json @@ -0,0 +1,3500 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp_dp_matrix_tp2_dp4", + "run_id": "20260709-152419_matrix", + "timestamp": "2026-07-09T15:24:26+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "6fce89c", + "git_dirty": "dirty", + "description": "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 2, + "dp": 4, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 2 --no-enable-flashinfer-autotune --port 30030 --data-parallel-size 4 " + }, + "scenarios": [ + { + "name": "c1_i1024_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.758116067008814, + "request_throughput": 1.050836072425456, + "input_token_throughput": 373.6773073544922, + "output_token_throughput": 79.65337428984957, + "total_token_throughput": 453.3306816443417, + "total_input_tokens": 1778, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 950.1402601948939, + "p50": 828.8744620222133, + "p90": 1316.3475427834783, + "p95": 1366.620513383532, + "p99": 1406.8388898635749 + }, + "ttft_ms": { + "mean": 159.96879799640737, + "p50": 121.72545900102705, + "p90": 273.4667629993055, + "p95": 319.0807249920908, + "p99": 355.5718945863191 + }, + "tpot_ms": { + "mean": 10.561727335790406, + "p50": 10.597711921482365, + "p90": 10.980906958036579, + "p95": 10.992017729171314, + "p99": 11.0009063460791 + }, + "itl_ms": { + "mean": 10.563021828840675, + "p50": 10.395356992376037, + "p90": 11.467909900238737, + "p95": 12.289851512468884, + "p99": 17.385886098782056 + }, + "gpu_memory": { + "peak_used_mb": 138264.0, + "avg_used_mb": 138239.14285714284, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1024_128.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 45.41872370798956, + "request_throughput": 14.091104895741893, + "input_token_throughput": 7372.620202911955, + "output_token_throughput": 919.5326638527569, + "total_token_throughput": 8292.152866764713, + "total_input_tokens": 334855, + "total_output_tokens": 41764, + "e2e_ms": { + "mean": 8857.588239907182, + "p50": 8342.36057351518, + "p90": 16634.871855200618, + "p95": 17800.523617253926, + "p99": 19385.132683189295 + }, + "ttft_ms": { + "mean": 682.8604170356357, + "p50": 574.7430620103842, + "p90": 1326.4753445982933, + "p95": 1787.9390910849907, + "p99": 1815.6773361668456 + }, + "tpot_ms": { + "mean": 131.98884306108397, + "p50": 139.4344896548215, + "p90": 154.4109195061627, + "p95": 159.76067048531368, + "p99": 191.59725015624025 + }, + "itl_ms": { + "mean": 127.26984535282504, + "p50": 80.08746500127017, + "p90": 209.82746919326036, + "p95": 320.7620121524087, + "p99": 333.84307554457337 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 138885.82142857142, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_128_1024_128.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.975539081002353, + "request_throughput": 0.8367445902740689, + "input_token_throughput": 297.5463763014589, + "output_token_throughput": 84.84590145379059, + "total_token_throughput": 382.3922777552495, + "total_input_tokens": 1778, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1193.4936490026303, + "p50": 1250.7776489946991, + "p90": 1856.4413986110594, + "p95": 1982.0633718161841, + "p99": 2082.5609503802843 + }, + "ttft_ms": { + "mean": 96.69496160349809, + "p50": 101.44926799694076, + "p90": 107.36800140584819, + "p95": 107.92980521218851, + "p99": 108.37924825726077 + }, + "tpot_ms": { + "mean": 11.009321936568274, + "p50": 11.267925303899592, + "p90": 11.515062654038317, + "p95": 11.568587976709885, + "p99": 11.61140823484714 + }, + "itl_ms": { + "mean": 10.923821774990376, + "p50": 10.853135987417772, + "p90": 11.97037369129248, + "p95": 12.5487451616209, + "p99": 15.062141394300852 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139312.0, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1024_256.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 51.14925201400183, + "request_throughput": 12.512401937467306, + "input_token_throughput": 6546.625548079085, + "output_token_throughput": 1654.8433587421603, + "total_token_throughput": 8201.468906821246, + "total_input_tokens": 334855, + "total_output_tokens": 84644, + "e2e_ms": { + "mean": 9769.298043770004, + "p50": 9525.109273497947, + "p90": 17658.252803297364, + "p95": 18928.978795289004, + "p99": 20487.34497815196 + }, + "ttft_ms": { + "mean": 455.66914050150444, + "p50": 347.1923780016368, + "p90": 1053.0307672161146, + "p95": 1302.987777524686, + "p99": 1324.667504846293 + }, + "tpot_ms": { + "mean": 72.5739601324665, + "p50": 75.14341008104658, + "p90": 88.30012603298934, + "p95": 92.16179340800115, + "p99": 105.85343872891032 + }, + "itl_ms": { + "mean": 70.98191771984622, + "p50": 29.792312008794397, + "p90": 188.3531395928003, + "p95": 192.85231040557846, + "p99": 200.51675533934022 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139312.4455128205, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_128_1024_256.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 6.343974668008741, + "request_throughput": 0.788149427079823, + "input_token_throughput": 280.2659362695851, + "output_token_throughput": 79.91835190589406, + "total_token_throughput": 360.18428817547914, + "total_input_tokens": 1778, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1267.0076036010869, + "p50": 1300.423595996108, + "p90": 2010.006586799864, + "p95": 2173.1115423957817, + "p99": 2303.595506872516 + }, + "ttft_ms": { + "mean": 95.3274762025103, + "p50": 94.37420600443147, + "p90": 112.07389460178092, + "p95": 113.70997279882431, + "p99": 115.01883535645902 + }, + "tpot_ms": { + "mean": 11.499225230518311, + "p50": 11.818655644599271, + "p90": 11.843597487057481, + "p95": 11.85012544028685, + "p99": 11.855347802870343 + }, + "itl_ms": { + "mean": 11.669650207161421, + "p50": 11.596686512348242, + "p90": 12.528862190083602, + "p95": 13.203412367147392, + "p99": 15.747690293937925 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1024_512.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 65.32000953899114, + "request_throughput": 9.797916511600752, + "input_token_throughput": 5126.377083581359, + "output_token_throughput": 2483.3431768434084, + "total_token_throughput": 7609.720260424768, + "total_input_tokens": 334855, + "total_output_tokens": 162212, + "e2e_ms": { + "mean": 12165.105242891741, + "p50": 11360.551028003101, + "p90": 22530.299117119284, + "p95": 24348.07764037541, + "p99": 26017.943218774457 + }, + "ttft_ms": { + "mean": 392.20384273189666, + "p50": 257.13926850585267, + "p90": 1188.2762587134494, + "p95": 1236.3547743909294, + "p99": 1486.3738604661194 + }, + "tpot_ms": { + "mean": 47.71833474125024, + "p50": 49.15205960397588, + "p90": 55.67882519789395, + "p95": 59.20219377536446, + "p99": 70.21163779017805 + }, + "itl_ms": { + "mean": 46.65614312299236, + "p50": 23.88978801900521, + "p90": 171.87037938274446, + "p95": 179.28580839652565, + "p99": 194.64230959885745 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_128_1024_512.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 18.318240207008785, + "request_throughput": 0.2729519835691934, + "input_token_throughput": 97.06172535720518, + "output_token_throughput": 83.57789736888702, + "total_token_throughput": 180.6396227260922, + "total_input_tokens": 1778, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 3661.845855199499, + "p50": 1542.1712879906408, + "p90": 7848.975591792259, + "p95": 8044.409900391475, + "p99": 8200.757347270846 + }, + "ttft_ms": { + "mean": 98.31969080260023, + "p50": 97.93389998958446, + "p90": 99.6642264071852, + "p95": 100.14198120916262, + "p99": 100.52418505074456 + }, + "tpot_ms": { + "mean": 11.7121814298879, + "p50": 11.669991084686192, + "p90": 11.861798871547494, + "p95": 11.897594997713366, + "p99": 11.926231898646064 + }, + "itl_ms": { + "mean": 11.675879083884157, + "p50": 11.550955998245627, + "p90": 12.462123500881717, + "p95": 12.981617015611846, + "p99": 15.28279174817726 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 90.39259965502424, + "request_throughput": 7.080225620709065, + "input_token_throughput": 3704.451484722709, + "output_token_throughput": 3578.744291397526, + "total_token_throughput": 7283.195776120236, + "total_input_tokens": 334855, + "total_output_tokens": 323492, + "e2e_ms": { + "mean": 16393.11319388412, + "p50": 15487.006908500916, + "p90": 30442.54000379297, + "p95": 32588.307497503407, + "p99": 34693.91080739064 + }, + "ttft_ms": { + "mean": 340.82878647768666, + "p50": 182.82538150378969, + "p90": 1101.2389173178235, + "p95": 1147.1085372351808, + "p99": 1259.63883144781 + }, + "tpot_ms": { + "mean": 32.786099067756574, + "p50": 33.33336248642041, + "p90": 38.46325583622135, + "p95": 39.48297626561873, + "p99": 50.717053920294475 + }, + "itl_ms": { + "mean": 31.838508019006326, + "p50": 20.308328996179625, + "p90": 51.352259801933535, + "p95": 67.75311281671735, + "p99": 189.77136611239962 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_128_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 66.40485374600394, + "request_throughput": 0.07529570081013673, + "input_token_throughput": 26.775151208084623, + "output_token_throughput": 84.73778169172787, + "total_token_throughput": 111.5129328998125, + "total_input_tokens": 1778, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 13278.26379040489, + "p50": 12816.862097999547, + "p90": 19899.61227281019, + "p95": 20062.545839411905, + "p99": 20192.892692693276 + }, + "ttft_ms": { + "mean": 95.99211560562253, + "p50": 98.21665400522761, + "p90": 104.68877339735627, + "p95": 105.11285619577393, + "p99": 105.45212243450806 + }, + "tpot_ms": { + "mean": 11.699930378375884, + "p50": 11.67477062993357, + "p90": 11.8023444723689, + "p95": 11.807400475164444, + "p99": 11.811445277400878 + }, + "itl_ms": { + "mean": 11.723782554422815, + "p50": 11.635283488431014, + "p90": 12.452087798737923, + "p95": 12.916478484112302, + "p99": 15.040395660907961 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 145.51546835299814, + "request_throughput": 4.3981578538953565, + "input_token_throughput": 2301.1642940095776, + "output_token_throughput": 4467.896144366185, + "total_token_throughput": 6769.060438375763, + "total_input_tokens": 334855, + "total_output_tokens": 650148, + "e2e_ms": { + "mean": 25564.97701632352, + "p50": 25074.268465992645, + "p90": 44908.9447277016, + "p95": 48968.45160179073, + "p99": 52638.13623400958 + }, + "ttft_ms": { + "mean": 282.0233969084711, + "p50": 119.06985749374144, + "p90": 974.8386680934345, + "p95": 990.9265333903022, + "p99": 1038.6707634397317 + }, + "tpot_ms": { + "mean": 25.236928190194135, + "p50": 26.040975281755923, + "p90": 27.663812289644483, + "p95": 28.25427095289195, + "p99": 31.35192731330995 + }, + "itl_ms": { + "mean": 24.927056339505796, + "p50": 20.130142016569152, + "p90": 29.90092160762288, + "p95": 49.22005442203953, + "p99": 175.35870163235813 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_128_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 136.8059436570038, + "request_throughput": 0.03654811966748949, + "input_token_throughput": 12.996511353759264, + "output_token_throughput": 86.04158332120376, + "total_token_throughput": 99.03809467496302, + "total_input_tokens": 1778, + "total_output_tokens": 11771, + "e2e_ms": { + "mean": 27359.510401409352, + "p50": 24717.59735699743, + "p90": 40834.48683540337, + "p95": 42290.37321120268, + "p99": 43455.082311842125 + }, + "ttft_ms": { + "mean": 98.28299639630131, + "p50": 102.85077299340628, + "p90": 111.57594778924249, + "p95": 114.23956138896756, + "p99": 116.37045226874761 + }, + "tpot_ms": { + "mean": 11.5920802460413, + "p50": 11.570362888125286, + "p90": 11.822217032060614, + "p95": 11.844249215126016, + "p99": 11.861874961578337 + }, + "itl_ms": { + "mean": 11.584727219787979, + "p50": 11.493468991830014, + "p90": 12.399183004163206, + "p95": 12.895709507574793, + "p99": 14.916132208600175 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 264.796780818986, + "request_throughput": 2.416947811905241, + "input_token_throughput": 1264.573530555515, + "output_token_throughput": 4961.163032031118, + "total_token_throughput": 6225.736562586633, + "total_input_tokens": 334855, + "total_output_tokens": 1313700, + "e2e_ms": { + "mean": 45632.76200614837, + "p50": 45892.894881995744, + "p90": 80464.02358178457, + "p95": 86442.33155561086, + "p99": 91949.11381047481 + }, + "ttft_ms": { + "mean": 265.10322853509933, + "p50": 111.98848400090355, + "p90": 881.7293376894668, + "p95": 901.8796355012455, + "p99": 946.0079092771048 + }, + "tpot_ms": { + "mean": 22.32576383160298, + "p50": 22.827393246859977, + "p90": 23.463852907171333, + "p95": 23.78087823866866, + "p99": 25.827674788164533 + }, + "itl_ms": { + "mean": 22.124532073000303, + "p50": 20.05979197565466, + "p90": 23.99035880225711, + "p95": 30.73016601265389, + "p99": 61.827885597012774 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_128_1024_4096.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.4484130880155135, + "request_throughput": 0.9176984048801556, + "input_token_throughput": 1829.8906193310304, + "output_token_throughput": 69.5615390899158, + "total_token_throughput": 1899.4521584209463, + "total_input_tokens": 9970, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 1088.2617171911988, + "p50": 1180.7936400000472, + "p90": 1500.1804889994673, + "p95": 1560.4593069991097, + "p99": 1608.6823613988236 + }, + "ttft_ms": { + "mean": 221.18209359468892, + "p50": 237.3110229964368, + "p90": 331.26449820119893, + "p95": 353.4522086090874, + "p99": 371.2023769353982 + }, + "tpot_ms": { + "mean": 11.62819634498916, + "p50": 11.65763964963844, + "p90": 11.887479798249698, + "p95": 11.952572407454307, + "p99": 12.004646494817992 + }, + "itl_ms": { + "mean": 11.591416058758707, + "p50": 11.50220850831829, + "p90": 12.546429494977929, + "p95": 13.778341589204501, + "p99": 15.210930389293933 + }, + "gpu_memory": { + "peak_used_mb": 139322.0, + "avg_used_mb": 139314.0, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_4096_128.jsonl" + }, + { + "name": "c64_i4096_o128", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 29.04734625099809, + "request_throughput": 11.01649690250119, + "input_token_throughput": 23259.85286785999, + "output_token_throughput": 721.0985753743436, + "total_token_throughput": 23980.95144323433, + "total_input_tokens": 675637, + "total_output_tokens": 20946, + "e2e_ms": { + "mean": 5607.2807608573385, + "p50": 5081.751556514064, + "p90": 10261.717318402958, + "p95": 11212.930104890256, + "p99": 12237.140612378425 + }, + "ttft_ms": { + "mean": 566.5797051917252, + "p50": 418.8897180138156, + "p90": 1271.789528624504, + "p95": 1790.0078742284677, + "p99": 2034.1907628765327 + }, + "tpot_ms": { + "mean": 83.97221397509522, + "p50": 85.30900120052598, + "p90": 102.1884308779788, + "p95": 111.6186297058448, + "p99": 188.61308130843136 + }, + "itl_ms": { + "mean": 78.22580327691779, + "p50": 24.40394448058214, + "p90": 193.86407711426727, + "p95": 204.7966962083592, + "p99": 265.26391156658065 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139315.09433962265, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_64_4096_128.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 6.499336033011787, + "request_throughput": 0.7693093532329647, + "input_token_throughput": 1534.0028503465317, + "output_token_throughput": 78.00796841782262, + "total_token_throughput": 1612.010818764354, + "total_input_tokens": 9970, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1298.4541277983226, + "p50": 1304.8905359755736, + "p90": 2055.9390532027464, + "p95": 2189.8132465954404, + "p99": 2296.9126013095956 + }, + "ttft_ms": { + "mean": 128.45938939717598, + "p50": 99.99181900639087, + "p90": 186.73655440798032, + "p95": 208.8984222093131, + "p99": 226.62791645037942 + }, + "tpot_ms": { + "mean": 11.608829012618783, + "p50": 11.749471700750291, + "p90": 11.840187970346939, + "p95": 11.866101362627823, + "p99": 11.886832076452531 + }, + "itl_ms": { + "mean": 11.652879141557746, + "p50": 11.607007996644825, + "p90": 12.479607487330213, + "p95": 13.407818757696077, + "p99": 15.573228679131718 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139316.5, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_4096_256.jsonl" + }, + { + "name": "c64_i4096_o256", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 37.625335443997756, + "request_throughput": 8.504907563582892, + "input_token_throughput": 17956.96947355142, + "output_token_throughput": 1101.0134397780778, + "total_token_throughput": 19057.9829133295, + "total_input_tokens": 675637, + "total_output_tokens": 41426, + "e2e_ms": { + "mean": 7132.9945801561735, + "p50": 6745.084112990298, + "p90": 13343.782691296656, + "p95": 14073.086364858318, + "p99": 15631.130007481263 + }, + "ttft_ms": { + "mean": 447.32724439654703, + "p50": 292.0360850112047, + "p90": 1235.394539491972, + "p95": 1290.521132601134, + "p99": 1667.2943222423783 + }, + "tpot_ms": { + "mean": 54.999285976148414, + "p50": 56.64399173060109, + "p90": 66.60206160047979, + "p95": 72.08972101296075, + "p99": 89.93224260194778 + }, + "itl_ms": { + "mean": 52.06885576994, + "p50": 20.320953015470877, + "p90": 183.50273961550556, + "p95": 190.4936825420009, + "p99": 198.8381153621594 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139316.71376811594, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_64_4096_256.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 6.4750854239973705, + "request_throughput": 0.7721905847711995, + "input_token_throughput": 1539.7480260337718, + "output_token_throughput": 78.30012529579963, + "total_token_throughput": 1618.0481513295715, + "total_input_tokens": 9970, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1293.4944579959847, + "p50": 1285.3574529872276, + "p90": 2067.5757369899657, + "p95": 2240.2964659850113, + "p99": 2378.473049181048 + }, + "ttft_ms": { + "mean": 124.96559479623102, + "p50": 99.54141901107505, + "p90": 182.4217675952241, + "p95": 210.03611879423258, + "p99": 232.1275997534394 + }, + "tpot_ms": { + "mean": 11.633153599071646, + "p50": 11.704481980368477, + "p90": 11.917797849731988, + "p95": 11.949951966215707, + "p99": 11.97567525940268 + }, + "itl_ms": { + "mean": 11.638103784917691, + "p50": 11.591274989768863, + "p90": 12.394777726149186, + "p95": 12.914686684962362, + "p99": 15.002317985636184 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_4096_512.jsonl" + }, + { + "name": "c64_i4096_o512", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 49.45528774798731, + "request_throughput": 6.470491115745719, + "input_token_throughput": 13661.572518653407, + "output_token_throughput": 1712.4559143515778, + "total_token_throughput": 15374.028433004985, + "total_input_tokens": 675637, + "total_output_tokens": 84690, + "e2e_ms": { + "mean": 9074.096850075239, + "p50": 8587.769280493376, + "p90": 16507.49469749572, + "p95": 17834.255616509472, + "p99": 18838.51776815718 + }, + "ttft_ms": { + "mean": 363.21495642187074, + "p50": 246.24972649326082, + "p90": 1080.8688154967967, + "p95": 1099.5816637849202, + "p99": 1280.63276574685 + }, + "tpot_ms": { + "mean": 32.92398170254894, + "p50": 34.29781718488103, + "p90": 39.79399375426641, + "p95": 40.95253786620741, + "p99": 51.14963246416302 + }, + "itl_ms": { + "mean": 33.051612037932166, + "p50": 16.917625995120034, + "p90": 55.95782760065051, + "p95": 179.05423378688283, + "p99": 191.79221595288254 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_64_4096_512.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 18.313489350985037, + "request_throughput": 0.27302279233482407, + "input_token_throughput": 544.4074479156392, + "output_token_throughput": 83.59957901292313, + "total_token_throughput": 628.0070269285624, + "total_input_tokens": 9970, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 3661.097132199211, + "p50": 1497.5465280003846, + "p90": 7919.612569594756, + "p95": 8164.683188794878, + "p99": 8360.739684154978 + }, + "ttft_ms": { + "mean": 103.66270020604134, + "p50": 101.5966720005963, + "p90": 111.75517840893008, + "p95": 113.91170821152627, + "p99": 115.63693205360323 + }, + "tpot_ms": { + "mean": 11.545292009534641, + "p50": 11.536920022783296, + "p90": 11.859975367734606, + "p95": 11.863926409369533, + "p99": 11.867087242677476 + }, + "itl_ms": { + "mean": 11.655896806018823, + "p50": 11.571258990443312, + "p90": 12.372287004836835, + "p95": 12.828694510972127, + "p99": 15.069960492837708 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_4096_1024.jsonl" + }, + { + "name": "c64_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 72.81607443501707, + "request_throughput": 4.394634048634085, + "input_token_throughput": 9278.679264740585, + "output_token_throughput": 2351.3764141844713, + "total_token_throughput": 11630.055678925057, + "total_input_tokens": 675637, + "total_output_tokens": 171218, + "e2e_ms": { + "mean": 13050.591836434796, + "p50": 13009.477967498242, + "p90": 23550.056274895916, + "p95": 25157.595743636197, + "p99": 26088.66373818193 + }, + "ttft_ms": { + "mean": 336.6113820809005, + "p50": 228.19460948812775, + "p90": 1073.0186230968684, + "p95": 1083.2181056088302, + "p99": 1200.0421604982694 + }, + "tpot_ms": { + "mean": 24.405185445081617, + "p50": 24.645095419248985, + "p90": 27.57983248187916, + "p95": 28.13234306312641, + "p99": 32.24075360975402 + }, + "itl_ms": { + "mean": 23.811828228222925, + "p50": 16.618399997241795, + "p90": 24.79075759765692, + "p95": 53.653452219441554, + "p99": 184.00424657273106 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_64_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 65.20599544400466, + "request_throughput": 0.0766800654748646, + "input_token_throughput": 152.90005055688002, + "output_token_throughput": 86.29574568541263, + "total_token_throughput": 239.19579624229263, + "total_input_tokens": 9970, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 13039.41647579777, + "p50": 12737.218773021596, + "p90": 19326.684689195827, + "p95": 19438.629691599635, + "p99": 19528.185693522682 + }, + "ttft_ms": { + "mean": 107.96508298953995, + "p50": 105.82048198557459, + "p90": 120.15626538195647, + "p95": 122.30626518139616, + "p99": 124.0262650209479 + }, + "tpot_ms": { + "mean": 11.543713116983527, + "p50": 11.544842900470172, + "p90": 11.710704597846302, + "p95": 11.75094882191123, + "p99": 11.783144201163173 + }, + "itl_ms": { + "mean": 11.500716007827993, + "p50": 11.416781999287196, + "p90": 12.267208803677931, + "p95": 12.714743409014773, + "p99": 15.013036065502092 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_4096_2048.jsonl" + }, + { + "name": "c64_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 117.00698560802266, + "request_throughput": 2.734879446189741, + "input_token_throughput": 5774.330451204056, + "output_token_throughput": 2846.0693886738927, + "total_token_throughput": 8620.39983987795, + "total_input_tokens": 675637, + "total_output_tokens": 333010, + "e2e_ms": { + "mean": 20535.820979047367, + "p50": 19770.545113497064, + "p90": 36951.179931193474, + "p95": 39745.30830668664, + "p99": 41227.42013381532 + }, + "ttft_ms": { + "mean": 297.6396630660929, + "p50": 127.86416050221305, + "p90": 920.5551686987747, + "p95": 1149.7123772627676, + "p99": 1194.8637982076616 + }, + "tpot_ms": { + "mean": 19.796835014518262, + "p50": 19.867121276273952, + "p90": 21.723155074566893, + "p95": 22.432242086481793, + "p99": 24.955436373794118 + }, + "itl_ms": { + "mean": 19.475918155825568, + "p50": 16.447670001070946, + "p90": 19.15072758274619, + "p95": 26.0455615032697, + "p99": 176.19469367782594 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_64_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 137.51461238099728, + "request_throughput": 0.036359772342934915, + "input_token_throughput": 72.50138605181222, + "output_token_throughput": 85.59817604973738, + "total_token_throughput": 158.0995621015496, + "total_input_tokens": 9970, + "total_output_tokens": 11771, + "e2e_ms": { + "mean": 27500.528350204695, + "p50": 25900.430945999688, + "p90": 40624.2244923953, + "p95": 42270.912868197775, + "p99": 43588.26356883976 + }, + "ttft_ms": { + "mean": 114.71709080506116, + "p50": 113.29897999530658, + "p90": 127.63377840747125, + "p95": 130.4347272089217, + "p99": 132.67548625008203 + }, + "tpot_ms": { + "mean": 11.660428161282525, + "p50": 11.607732123008983, + "p90": 11.838800223548626, + "p95": 11.866398650268481, + "p99": 11.888477391644368 + }, + "itl_ms": { + "mean": 11.637672173722205, + "p50": 11.551771996892057, + "p90": 12.423009000485763, + "p95": 12.856199762609322, + "p99": 14.940864442905877 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_4096_4096.jsonl" + }, + { + "name": "c64_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 214.53852150400053, + "request_throughput": 1.4915736239658617, + "input_token_throughput": 3149.2572767981965, + "output_token_throughput": 3050.9485914761212, + "total_token_throughput": 6200.205868274318, + "total_input_tokens": 675637, + "total_output_tokens": 654546, + "e2e_ms": { + "mean": 36687.187690068735, + "p50": 36015.92721400084, + "p90": 65686.69284979116, + "p95": 72324.60426831385, + "p99": 75036.71893385006 + }, + "ttft_ms": { + "mean": 274.7341550009878, + "p50": 121.60490201495122, + "p90": 876.4140803163173, + "p95": 900.9255425946321, + "p99": 999.7351078558131 + }, + "tpot_ms": { + "mean": 17.93428980380427, + "p50": 18.12657454228293, + "p90": 18.85609416246715, + "p95": 19.27635751648694, + "p99": 20.036325411799265 + }, + "itl_ms": { + "mean": 17.821273568422196, + "p50": 16.586008510785177, + "p90": 17.627767586964186, + "p95": 19.988030730746686, + "p99": 55.21536503219978 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_64_4096_4096.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.712559448002139, + "request_throughput": 0.8752644144033646, + "input_token_throughput": 7481.410108554199, + "output_token_throughput": 66.34504261177504, + "total_token_throughput": 7547.755151165974, + "total_input_tokens": 42738, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 1140.7567776041105, + "p50": 1153.5738079983275, + "p90": 1741.2836964067537, + "p95": 1856.5267082012724, + "p99": 1948.7211176368874 + }, + "ttft_ms": { + "mean": 320.95666300156154, + "p50": 255.25449600536376, + "p90": 575.1963478047401, + "p95": 599.0957394067663, + "p99": 618.2152526883874 + }, + "tpot_ms": { + "mean": 10.996233152215892, + "p50": 11.146897429644614, + "p90": 11.40574593279013, + "p95": 11.481190116210913, + "p99": 11.54154546294754 + }, + "itl_ms": { + "mean": 10.959262572126812, + "p50": 11.04035900789313, + "p90": 11.958558010519482, + "p95": 12.407819404324982, + "p99": 14.681235762254797 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_16384_128.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 37.1104527740099, + "request_throughput": 4.311453729070509, + "input_token_throughput": 34791.24892014867, + "output_token_throughput": 284.71762563349375, + "total_token_throughput": 35075.96654578216, + "total_input_tokens": 1291119, + "total_output_tokens": 10566, + "e2e_ms": { + "mean": 7222.81702600103, + "p50": 6927.245188999223, + "p90": 12521.500642600584, + "p95": 13755.1466933437, + "p99": 15245.925571036641 + }, + "ttft_ms": { + "mean": 956.2753173306191, + "p50": 684.0546940075001, + "p90": 1860.2323261031415, + "p95": 3298.2727444323245, + "p99": 4051.797657485294 + }, + "tpot_ms": { + "mean": 107.20667041146244, + "p50": 102.95063398139787, + "p90": 146.80821743508776, + "p95": 186.59272597506057, + "p99": 376.91161463852035 + }, + "itl_ms": { + "mean": 96.37079262274476, + "p50": 15.931423506117426, + "p90": 315.78448468935676, + "p95": 334.5220921881269, + "p99": 419.7137621362343 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.38076923077, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_32_16384_128.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 7.071820794022642, + "request_throughput": 0.7070314909883152, + "input_token_throughput": 6043.422372371723, + "output_token_throughput": 71.69299318621516, + "total_token_throughput": 6115.115365557938, + "total_input_tokens": 42738, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1412.5724327983335, + "p50": 1449.3207980121952, + "p90": 2229.337644588668, + "p95": 2447.118662792491, + "p99": 2621.343477355549 + }, + "ttft_ms": { + "mean": 256.48318639723584, + "p50": 241.63091200171039, + "p90": 402.00019779149443, + "p95": 448.9649123861454, + "p99": 486.53668406186625 + }, + "tpot_ms": { + "mean": 11.51924006850755, + "p50": 11.537074755401687, + "p90": 11.659376842578114, + "p95": 11.662781071134695, + "p99": 11.665504453979961 + }, + "itl_ms": { + "mean": 11.53735575050547, + "p50": 11.457617016276345, + "p90": 12.345440016360953, + "p95": 13.56259299791418, + "p99": 14.976673992350698 + }, + "gpu_memory": { + "peak_used_mb": 139324.0, + "avg_used_mb": 139317.5, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_16384_256.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 35.65484240101068, + "request_throughput": 4.487469000717406, + "input_token_throughput": 36211.603054607855, + "output_token_throughput": 623.0289773871028, + "total_token_throughput": 36834.63203199495, + "total_input_tokens": 1291119, + "total_output_tokens": 22214, + "e2e_ms": { + "mean": 6749.376519124962, + "p50": 6248.172036983306, + "p90": 11905.916951200925, + "p95": 12780.120356346015, + "p99": 13936.329891999774 + }, + "ttft_ms": { + "mean": 680.0791634877896, + "p50": 551.1939769930905, + "p90": 1328.4132753062295, + "p95": 1913.910021353506, + "p99": 2384.965779259219 + }, + "tpot_ms": { + "mean": 47.33705474009782, + "p50": 46.58911146607264, + "p90": 59.486993915876845, + "p95": 71.66590644790797, + "p99": 147.9650174480042 + }, + "itl_ms": { + "mean": 44.06801878353641, + "p50": 14.36969700444024, + "p90": 177.0988059870433, + "p95": 292.33303774526576, + "p99": 365.75512995623285 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139370.66666666666, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_32_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 6.524697318003746, + "request_throughput": 0.7663190729481636, + "input_token_throughput": 6550.188907931723, + "output_token_throughput": 77.7047539969438, + "total_token_throughput": 6627.893661928667, + "total_input_tokens": 42738, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1303.133088402683, + "p50": 1312.3146350262687, + "p90": 2065.8049791993108, + "p95": 2227.29969760403, + "p99": 2356.495472327806 + }, + "ttft_ms": { + "mean": 139.0515518025495, + "p50": 114.63064100826159, + "p90": 185.5096512066666, + "p95": 188.539885613136, + "p99": 190.9640731383115 + }, + "tpot_ms": { + "mean": 11.553020613458711, + "p50": 11.625021600048058, + "p90": 11.720145275427008, + "p95": 11.73107260839001, + "p99": 11.739814474760411 + }, + "itl_ms": { + "mean": 11.593969956117373, + "p50": 11.583370011067018, + "p90": 12.42344960628543, + "p95": 13.154048309661448, + "p99": 15.070866440946702 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139408.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_16384_512.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 42.4050869079947, + "request_throughput": 3.7731322269695653, + "input_token_throughput": 30447.26692345449, + "output_token_throughput": 952.4800665456296, + "total_token_throughput": 31399.746990000116, + "total_input_tokens": 1291119, + "total_output_tokens": 40390, + "e2e_ms": { + "mean": 7937.473418123591, + "p50": 7231.936971991672, + "p90": 14335.53286208771, + "p95": 16413.951087038724, + "p99": 18769.122077563075 + }, + "ttft_ms": { + "mean": 538.1829345238657, + "p50": 374.5491640002001, + "p90": 1393.5831179056543, + "p95": 1520.067489783105, + "p99": 2259.12406618183 + }, + "tpot_ms": { + "mean": 30.214154563376304, + "p50": 29.921590453918636, + "p90": 38.20460801400256, + "p95": 45.07352797602294, + "p99": 57.04345311538565 + }, + "itl_ms": { + "mean": 29.449055698720237, + "p50": 14.489118999335915, + "p90": 33.242931007407606, + "p95": 144.51645698864013, + "p99": 316.92803697660565 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139408.4471830986, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_32_16384_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 18.25210122601129, + "request_throughput": 0.2739410623514645, + "input_token_throughput": 2341.5386245553777, + "output_token_throughput": 83.88075329201841, + "total_token_throughput": 2425.419377847396, + "total_input_tokens": 42738, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 3648.9271302009, + "p50": 1574.4030399946496, + "p90": 7676.770175999263, + "p95": 7809.608873003162, + "p99": 7915.879830606282 + }, + "ttft_ms": { + "mean": 188.47413159674034, + "p50": 166.91147000528872, + "p90": 289.86948799574753, + "p95": 322.9151319887023, + "p99": 349.35164718306623 + }, + "tpot_ms": { + "mean": 11.311722712704132, + "p50": 11.38147915044101, + "p90": 11.636269275296959, + "p95": 11.637638455246188, + "p99": 11.638733799205573 + }, + "itl_ms": { + "mean": 11.338147827002084, + "p50": 11.296455500996672, + "p90": 12.177108001196757, + "p95": 12.575102984555997, + "p99": 15.079722506925464 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139409.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 63.50027614500141, + "request_throughput": 2.519674081962159, + "input_token_throughput": 20332.49425643063, + "output_token_throughput": 1337.5374904835876, + "total_token_throughput": 21670.031746914217, + "total_input_tokens": 1291119, + "total_output_tokens": 84934, + "e2e_ms": { + "mean": 11657.908230049725, + "p50": 11766.50226500351, + "p90": 19649.950374499895, + "p95": 21919.155883429627, + "p99": 22977.80829768249 + }, + "ttft_ms": { + "mean": 485.63056064431294, + "p50": 246.81737599894404, + "p90": 1364.2390364984747, + "p95": 1871.965904346143, + "p99": 2404.3688034138167 + }, + "tpot_ms": { + "mean": 22.945847302867808, + "p50": 21.532263212585303, + "p90": 25.27922502447743, + "p95": 27.86078809842337, + "p99": 72.2763683168145 + }, + "itl_ms": { + "mean": 21.100112798235703, + "p50": 14.256738999392837, + "p90": 15.739107580156997, + "p95": 52.42427446501096, + "p99": 297.65237216226524 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139409.41842105263, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_32_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 65.72091866700794, + "request_throughput": 0.07607927736576227, + "input_token_throughput": 650.2952312115896, + "output_token_throughput": 85.61961874742886, + "total_token_throughput": 735.9148499590184, + "total_input_tokens": 42738, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 13142.26782359765, + "p50": 12754.602688015439, + "p90": 19704.28160718293, + "p95": 19985.512879578164, + "p99": 20210.49789749435 + }, + "ttft_ms": { + "mean": 175.36014119395986, + "p50": 185.10537198744714, + "p90": 234.92859159014188, + "p95": 249.45631479495202, + "p99": 261.0784933588002 + }, + "tpot_ms": { + "mean": 11.542516854607184, + "p50": 11.560659686032267, + "p90": 11.633962648627595, + "p95": 11.641122501803924, + "p99": 11.646850384344985 + }, + "itl_ms": { + "mean": 11.532248108137457, + "p50": 11.443908995715901, + "p90": 12.273180313059129, + "p95": 12.68829941545846, + "p99": 14.829100963252131 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139409.5, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 102.62741485901643, + "request_throughput": 1.5590376140702626, + "input_token_throughput": 12580.644282754898, + "output_token_throughput": 1785.4683395437926, + "total_token_throughput": 14366.11262229869, + "total_input_tokens": 1291119, + "total_output_tokens": 183238, + "e2e_ms": { + "mean": 18450.049960798606, + "p50": 19650.882197995088, + "p90": 29807.86607058253, + "p95": 31359.527737388264, + "p99": 33430.94244295469 + }, + "ttft_ms": { + "mean": 336.451010986093, + "p50": 182.73530549777206, + "p90": 1068.0653378920397, + "p95": 1089.1261353419395, + "p99": 1111.2576313567115 + }, + "tpot_ms": { + "mean": 15.831536450525308, + "p50": 16.035249127389623, + "p90": 16.929491397301096, + "p95": 17.467017443684117, + "p99": 19.69697846989029 + }, + "itl_ms": { + "mean": 15.838644518932316, + "p50": 14.241769997170195, + "p90": 15.130802989006042, + "p95": 15.871898998739198, + "p99": 60.515860002488125 + }, + "gpu_memory": { + "peak_used_mb": 139684.0, + "avg_used_mb": 139479.6037037037, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_32_16384_2048.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 11.397514649986988, + "request_throughput": 0.43869213188558687, + "input_token_throughput": 15249.81588860677, + "output_token_throughput": 33.25286359692748, + "total_token_throughput": 15283.068752203697, + "total_input_tokens": 173810, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 2278.139853401808, + "p50": 2724.3857540015597, + "p90": 3841.2523665989283, + "p95": 4059.395702800248, + "p99": 4233.910371761303 + }, + "ttft_ms": { + "mean": 1429.4971552037168, + "p50": 1554.4339650077745, + "p90": 2710.1020148023963, + "p95": 2811.6455034061796, + "p99": 2892.880294289207 + }, + "tpot_ms": { + "mean": 11.473724474729648, + "p50": 11.380048349868352, + "p90": 11.930784581619424, + "p95": 12.084340916035751, + "p99": 12.207185983568813 + }, + "itl_ms": { + "mean": 11.68852257848584, + "p50": 11.67310102027841, + "p90": 12.547637592069805, + "p95": 13.033487010397947, + "p99": 15.609450997435488 + }, + "gpu_memory": { + "peak_used_mb": 139706.0, + "avg_used_mb": 139524.5, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_65536_128.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 46.495504008024, + "request_throughput": 0.8602982342786728, + "input_token_throughput": 31649.769830348378, + "output_token_throughput": 55.231146640690795, + "total_token_throughput": 31705.000976989068, + "total_input_tokens": 1471572, + "total_output_tokens": 2568, + "e2e_ms": { + "mean": 9193.44558102457, + "p50": 8837.237635001657, + "p90": 17342.221115305438, + "p95": 17992.55950945808, + "p99": 18685.863364119432 + }, + "ttft_ms": { + "mean": 2642.309532523359, + "p50": 2382.497742000851, + "p90": 4619.818356982434, + "p95": 5855.872693145648, + "p99": 7061.538508331286 + }, + "tpot_ms": { + "mean": 115.3907831548218, + "p50": 107.64835458233807, + "p90": 174.2567664918026, + "p95": 327.7387829276752, + "p99": 399.9186071318399 + }, + "itl_ms": { + "mean": 107.2191137831152, + "p50": 11.844835506053641, + "p90": 351.82774128916213, + "p95": 420.32483855873573, + "p99": 448.0478705113637 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139654.87837837837, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_8_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 10.354734632011969, + "request_throughput": 0.48287089700419283, + "input_token_throughput": 16785.558121659753, + "output_token_throughput": 48.963108956225156, + "total_token_throughput": 16834.521230615977, + "total_input_tokens": 173810, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 2069.1889655892737, + "p50": 2424.590051989071, + "p90": 3329.392332781572, + "p95": 3540.820776380133, + "p99": 3709.9635312589817 + }, + "ttft_ms": { + "mean": 902.4706331896596, + "p50": 369.5778209948912, + "p90": 2014.7223625914194, + "p95": 2181.4932107925415, + "p99": 2314.909889353439 + }, + "tpot_ms": { + "mean": 11.550378226712771, + "p50": 11.603183148636614, + "p90": 11.927110441010495, + "p95": 11.945248954555396, + "p99": 11.959759765391317 + }, + "itl_ms": { + "mean": 11.880563763683936, + "p50": 11.901292018592358, + "p90": 12.793024972779676, + "p95": 13.358539014006965, + "p99": 15.600708706188017 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139692.5, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_65536_256.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 44.64320889499504, + "request_throughput": 0.895992940249517, + "input_token_throughput": 32962.953076721555, + "output_token_throughput": 117.73347234878653, + "total_token_throughput": 33080.68654907034, + "total_input_tokens": 1471572, + "total_output_tokens": 5256, + "e2e_ms": { + "mean": 8621.679103097995, + "p50": 7934.184520490817, + "p90": 15938.640361695434, + "p95": 16550.715314448462, + "p99": 18052.147336646158 + }, + "ttft_ms": { + "mean": 1800.3800100006629, + "p50": 1788.63854049996, + "p90": 3144.015901285457, + "p95": 4029.433374495418, + "p99": 4806.74102140154 + }, + "tpot_ms": { + "mean": 54.446565760702576, + "p50": 55.22089271177891, + "p90": 92.74740785803097, + "p95": 101.30015664374284, + "p99": 122.7065518255012 + }, + "itl_ms": { + "mean": 53.78479873170394, + "p50": 11.341089993948117, + "p90": 325.2451349981129, + "p95": 342.1970723953564, + "p99": 385.69993141572365 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139694.57638888888, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_8_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 8.023412760987412, + "request_throughput": 0.6231762155266044, + "input_token_throughput": 21662.851604135823, + "output_token_throughput": 63.19006825439769, + "total_token_throughput": 21726.04167239022, + "total_input_tokens": 173810, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1602.9774336027913, + "p50": 1639.2050010035746, + "p90": 2489.9309328000527, + "p95": 2564.6468784019817, + "p99": 2624.419634883525 + }, + "ttft_ms": { + "mean": 501.87500101164915, + "p50": 346.59205802017823, + "p90": 1054.3755090038758, + "p95": 1276.3666950049808, + "p99": 1453.959643805865 + }, + "tpot_ms": { + "mean": 11.056220900028002, + "p50": 11.125115478631763, + "p90": 11.556862828072946, + "p95": 11.680376113908292, + "p99": 11.77918674257657 + }, + "itl_ms": { + "mean": 11.189579554810756, + "p50": 11.18360050895717, + "p90": 12.12988500483334, + "p95": 12.547136211651376, + "p99": 14.437380578601728 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139696.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_65536_512.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 40.84438401600346, + "request_throughput": 0.9793268025373423, + "input_token_throughput": 36028.74753658705, + "output_token_throughput": 272.8404471869036, + "total_token_throughput": 36301.58798377395, + "total_input_tokens": 1471572, + "total_output_tokens": 11144, + "e2e_ms": { + "mean": 7629.467499878228, + "p50": 6605.449742506607, + "p90": 13734.331959995325, + "p95": 14481.515541051333, + "p99": 18250.23612089688 + }, + "ttft_ms": { + "mean": 1260.9102477501438, + "p50": 736.1879230156774, + "p90": 2563.182401913219, + "p95": 3168.1889003870274, + "p99": 5671.910491968155 + }, + "tpot_ms": { + "mean": 24.558571631178896, + "p50": 19.7090078711015, + "p90": 36.660957203864, + "p95": 51.5223379378131, + "p99": 85.55504920109868 + }, + "itl_ms": { + "mean": 23.329978548782595, + "p50": 11.118012014776468, + "p90": 12.710754189174624, + "p95": 23.5887666989584, + "p99": 342.11543018638616 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139696.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_8_65536_512.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 18.417840418987907, + "request_throughput": 0.271475910652654, + "input_token_throughput": 9437.045606107557, + "output_token_throughput": 83.12592384184265, + "total_token_throughput": 9520.171529949399, + "total_input_tokens": 173810, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 3681.792501191376, + "p50": 1626.8658789922483, + "p90": 7768.170245189685, + "p95": 8015.790109592489, + "p99": 8213.886001114734 + }, + "ttft_ms": { + "mean": 255.57148939697072, + "p50": 295.48558199894615, + "p90": 365.17806919291615, + "p95": 375.57831059093587, + "p99": 383.89850370935164 + }, + "tpot_ms": { + "mean": 11.11591223133066, + "p50": 10.96062714980835, + "p90": 11.564500371784172, + "p95": 11.57499151231189, + "p99": 11.583384424734062 + }, + "itl_ms": { + "mean": 11.29259778707056, + "p50": 11.243843997363001, + "p90": 12.119136797264218, + "p95": 12.575050792656842, + "p99": 14.376279006246476 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139696.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 62.338421098014805, + "request_throughput": 0.6416588565358101, + "input_token_throughput": 23606.18017075288, + "output_token_throughput": 351.24405806770244, + "total_token_throughput": 23957.424228820582, + "total_input_tokens": 1471572, + "total_output_tokens": 21896, + "e2e_ms": { + "mean": 11157.950221427018, + "p50": 11160.75557350996, + "p90": 20058.121949192722, + "p95": 21105.82270954037, + "p99": 21507.370753068535 + }, + "ttft_ms": { + "mean": 962.3508537973976, + "p50": 418.21416800667066, + "p90": 2749.005644590943, + "p95": 2850.090325153724, + "p99": 2967.291713570012 + }, + "tpot_ms": { + "mean": 19.43057001452085, + "p50": 18.126048372799833, + "p90": 26.67870498145342, + "p95": 29.437979978438662, + "p99": 36.682113960456 + }, + "itl_ms": { + "mean": 18.854458797366757, + "p50": 11.316222997265868, + "p90": 12.648795178392902, + "p95": 13.506781545584087, + "p99": 334.8691878540558 + }, + "gpu_memory": { + "peak_used_mb": 139708.0, + "avg_used_mb": 139696.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_8_65536_1024.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 15.532598288002191, + "request_throughput": 0.32190364466337473, + "input_token_throughput": 23847.780849783587, + "output_token_throughput": 24.400296265483803, + "total_token_throughput": 23872.18114604907, + "total_input_tokens": 370418, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 3104.6353054058272, + "p50": 4526.820452010725, + "p90": 4888.86635240051, + "p95": 4990.354587201728, + "p99": 5071.545175042702 + }, + "ttft_ms": { + "mean": 2286.41628260375, + "p50": 3430.2771130169276, + "p90": 3812.3717041977216, + "p95": 3831.7083316040225, + "p99": 3847.1776335290633 + }, + "tpot_ms": { + "mean": 11.064339925313314, + "p50": 11.05356198367047, + "p90": 11.613551280170213, + "p95": 11.651908540225122, + "p99": 11.68259434826905 + }, + "itl_ms": { + "mean": 11.555975477430195, + "p50": 11.574425516300835, + "p90": 12.51095628249459, + "p95": 13.191249890951436, + "p99": 15.23967941524461 + }, + "gpu_memory": { + "peak_used_mb": 140228.0, + "avg_used_mb": 139805.56818181818, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_131072_128.jsonl" + }, + { + "name": "c4_i131072_o128", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 27.13875742800883, + "request_throughput": 0.7369534162739079, + "input_token_throughput": 54328.2426953833, + "output_token_throughput": 41.96949705679905, + "total_token_throughput": 54370.2121924401, + "total_input_tokens": 1474401, + "total_output_tokens": 1139, + "e2e_ms": { + "mean": 5273.325644248689, + "p50": 5078.873286503949, + "p90": 9009.436314093182, + "p95": 9530.159654750605, + "p99": 9800.838134941296 + }, + "ttft_ms": { + "mean": 2648.468313649937, + "p50": 2407.475689004059, + "p90": 5161.319839497446, + "p95": 5381.790485109378, + "p99": 6485.259353810396 + }, + "tpot_ms": { + "mean": 63.46264448269876, + "p50": 27.108041290587686, + "p90": 165.854421498146, + "p95": 234.23061024928757, + "p99": 254.36114444131087 + }, + "itl_ms": { + "mean": 49.66539044279866, + "p50": 11.527488008141518, + "p90": 63.46408920944667, + "p95": 397.0826292119454, + "p99": 499.1101731651 + }, + "gpu_memory": { + "peak_used_mb": 140228.0, + "avg_used_mb": 140084.00862068965, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_4_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 10.823737689002883, + "request_throughput": 0.46194763247820503, + "input_token_throughput": 34222.74362546235, + "output_token_throughput": 46.84148993328999, + "total_token_throughput": 34269.58511539564, + "total_input_tokens": 370418, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 2162.711608601967, + "p50": 1606.9780830002856, + "p90": 3969.7245941963047, + "p95": 4514.326268597506, + "p99": 4950.007608118467 + }, + "ttft_ms": { + "mean": 1053.5080416011624, + "p50": 547.0652380026877, + "p90": 2534.259543009103, + "p95": 3143.29669700819, + "p99": 3630.5264202074613 + }, + "tpot_ms": { + "mean": 11.109271387096143, + "p50": 10.817969112548793, + "p90": 11.812963536160286, + "p95": 11.900103468407055, + "p99": 11.96981541420447 + }, + "itl_ms": { + "mean": 11.481921894436772, + "p50": 11.444862990174443, + "p90": 12.47322719427757, + "p95": 13.036605808883904, + "p99": 15.39386351883877 + }, + "gpu_memory": { + "peak_used_mb": 140228.0, + "avg_used_mb": 140216.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_131072_256.jsonl" + }, + { + "name": "c4_i131072_o256", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.31919323001057, + "request_throughput": 0.5086574356448113, + "input_token_throughput": 37498.25158860727, + "output_token_throughput": 64.77752442936672, + "total_token_throughput": 37563.02911303664, + "total_input_tokens": 1474401, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 7621.5941461516195, + "p50": 5433.910031002597, + "p90": 15992.410031994228, + "p95": 19293.47165230138, + "p99": 21326.096049646727 + }, + "ttft_ms": { + "mean": 2168.9784800997586, + "p50": 1628.2644945022184, + "p90": 4288.226667698473, + "p95": 4431.1157627482335, + "p99": 6485.733761349398 + }, + "tpot_ms": { + "mean": 38.017727809748095, + "p50": 26.924098604547815, + "p90": 76.59060437403086, + "p95": 82.59105530018105, + "p99": 93.3094082225064 + }, + "itl_ms": { + "mean": 45.02534406360306, + "p50": 11.698337984853424, + "p90": 62.21985211595896, + "p95": 360.9232887451071, + "p99": 418.3628251054324 + }, + "gpu_memory": { + "peak_used_mb": 140228.0, + "avg_used_mb": 140216.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_4_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 7.443610455986345, + "request_throughput": 0.6717170423633426, + "input_token_throughput": 49763.21667962893, + "output_token_throughput": 68.11210809564294, + "total_token_throughput": 49831.328787724575, + "total_input_tokens": 370418, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1487.1138908027206, + "p50": 1618.9608690037858, + "p90": 2149.4011818023864, + "p95": 2243.5973944026046, + "p99": 2318.954364482779 + }, + "ttft_ms": { + "mean": 391.2680192035623, + "p50": 547.8159560007043, + "p90": 570.8218260144349, + "p95": 574.5017840177752, + "p99": 577.4457504204474 + }, + "tpot_ms": { + "mean": 10.888728879290689, + "p50": 10.649841338682782, + "p90": 11.532310937386393, + "p95": 11.536287318988197, + "p99": 11.539468424269643 + }, + "itl_ms": { + "mean": 11.320199055728514, + "p50": 11.297071498120204, + "p90": 12.151737086242065, + "p95": 12.580379500286654, + "p99": 15.034107333340218 + }, + "gpu_memory": { + "peak_used_mb": 140228.0, + "avg_used_mb": 140216.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_131072_512.jsonl" + }, + { + "name": "c4_i131072_o512", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.07605390899698, + "request_throughput": 0.464295082419858, + "input_token_throughput": 34227.856690746055, + "output_token_throughput": 136.38668046083328, + "total_token_throughput": 34364.24337120689, + "total_input_tokens": 1474401, + "total_output_tokens": 5875, + "e2e_ms": { + "mean": 8028.2892768489555, + "p50": 6942.520142998546, + "p90": 13616.571091485093, + "p95": 13969.639348344936, + "p99": 14391.104528077703 + }, + "ttft_ms": { + "mean": 1588.4933723005815, + "p50": 721.3263924932107, + "p90": 4283.278025590699, + "p95": 5162.584823240469, + "p99": 5751.134359843271 + }, + "tpot_ms": { + "mean": 24.25930484325447, + "p50": 21.142737899969127, + "p90": 43.33457116035198, + "p95": 43.94312149891555, + "p99": 51.734927765984985 + }, + "itl_ms": { + "mean": 22.61147524245899, + "p50": 11.681557996780612, + "p90": 13.17090202064719, + "p95": 17.04175149643561, + "p99": 376.24420104693854 + }, + "gpu_memory": { + "peak_used_mb": 140228.0, + "avg_used_mb": 140216.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_4_131072_512.jsonl" + }, + { + "name": "c1_i262144_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 33.3968951869756, + "request_throughput": 0.14971451603530925, + "input_token_throughput": 22865.418947621467, + "output_token_throughput": 11.34836031547644, + "total_token_throughput": 22876.76730793694, + "total_input_tokens": 763634, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 6677.662302803947, + "p50": 6592.009656014852, + "p90": 12769.75543780718, + "p95": 14582.278500404207, + "p99": 16032.296950481832 + }, + "ttft_ms": { + "mean": 5896.153790998505, + "p50": 6455.220932024531, + "p90": 11764.465710002696, + "p95": 13444.272032997103, + "p99": 14788.117091392633 + }, + "tpot_ms": { + "mean": 9.882022091738845, + "p50": 10.501844686030772, + "p90": 11.01214369704212, + "p95": 11.086593334589374, + "p99": 11.146153044627178 + }, + "itl_ms": { + "mean": 11.628826648856853, + "p50": 11.615761992288753, + "p90": 12.903074515634216, + "p95": 14.062745511182584, + "p99": 16.517276962986216 + }, + "gpu_memory": { + "peak_used_mb": 140744.0, + "avg_used_mb": 140300.84016393442, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_262144_128.jsonl" + }, + { + "name": "c2_i262144_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 36.61425120799686, + "request_throughput": 0.27311769789288814, + "input_token_throughput": 38069.90322105947, + "output_token_throughput": 20.64769796070234, + "total_token_throughput": 38090.55091902017, + "total_input_tokens": 1393901, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 6665.653762695729, + "p50": 7193.123516000924, + "p90": 10316.113390692044, + "p95": 10320.338272850495, + "p99": 10323.718178577255 + }, + "ttft_ms": { + "mean": 4480.795355394366, + "p50": 4243.354582009488, + "p90": 9116.933727281867, + "p95": 9292.653828128821, + "p99": 9433.229908806388 + }, + "tpot_ms": { + "mean": 24.302159523430515, + "p50": 10.971240596767418, + "p90": 72.93804791286638, + "p95": 74.40089603748876, + "p99": 75.57117453718669 + }, + "itl_ms": { + "mean": 32.082317728380346, + "p50": 11.63253202685155, + "p90": 14.834656001767144, + "p95": 24.26114000263624, + "p99": 520.8656103874092 + }, + "gpu_memory": { + "peak_used_mb": 140744.0, + "avg_used_mb": 140533.4631147541, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_2_262144_128.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 27.865258064994123, + "request_throughput": 0.1794349073795687, + "input_token_throughput": 27404.519212377912, + "output_token_throughput": 18.194699608288264, + "total_token_throughput": 27422.7139119862, + "total_input_tokens": 763634, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 5571.334156603552, + "p50": 6127.82426000922, + "p90": 9963.428542204201, + "p95": 10356.68986660894, + "p99": 10671.298926132731 + }, + "ttft_ms": { + "mean": 4494.808082003146, + "p50": 4998.44012001995, + "p90": 8390.57087499532, + "p95": 8943.433845997788, + "p99": 9385.724222799763 + }, + "tpot_ms": { + "mean": 10.07351614397731, + "p50": 10.913098973415277, + "p90": 11.340758431304447, + "p95": 11.430213398637076, + "p99": 11.501777372503177 + }, + "itl_ms": { + "mean": 11.57473422585666, + "p50": 11.503823014209047, + "p90": 12.946098006796092, + "p95": 14.1087103926111, + "p99": 15.470532923936846 + }, + "gpu_memory": { + "peak_used_mb": 141258.0, + "avg_used_mb": 140712.5613207547, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_262144_256.jsonl" + }, + { + "name": "c2_i262144_o256", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 33.1799936350144, + "request_throughput": 0.30138643515130564, + "input_token_throughput": 42010.285334384, + "output_token_throughput": 42.07354634712227, + "total_token_throughput": 42052.358880731124, + "total_input_tokens": 1393901, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 6437.8450697957305, + "p50": 6647.003061501891, + "p90": 11981.908666883828, + "p95": 12049.746989947744, + "p99": 12104.017648398876 + }, + "ttft_ms": { + "mean": 2785.8514580962947, + "p50": 633.666234993143, + "p90": 7290.944269279133, + "p95": 9153.02144213783, + "p99": 10642.683180424794 + }, + "tpot_ms": { + "mean": 24.921164712144638, + "p50": 11.55330469214383, + "p90": 55.22409717675922, + "p95": 66.1042117439868, + "p99": 74.80830339776891 + }, + "itl_ms": { + "mean": 27.87743504576826, + "p50": 11.707381505402736, + "p90": 14.250949586858049, + "p95": 23.011322165257297, + "p99": 423.4108724640106 + }, + "gpu_memory": { + "peak_used_mb": 141258.0, + "avg_used_mb": 140860.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_2_262144_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 38.84872146099224, + "request_throughput": 0.12870436431274757, + "input_token_throughput": 26404.421083200315, + "output_token_throughput": 9.755790814906266, + "total_token_throughput": 26414.176874015222, + "total_input_tokens": 1025778, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 7768.2642875937745, + "p50": 6653.598174976651, + "p90": 15560.610908200033, + "p95": 18082.004304596918, + "p99": 20099.119021714432 + }, + "ttft_ms": { + "mean": 7011.266513593728, + "p50": 6505.533494986594, + "p90": 14698.492897400867, + "p95": 17139.21816620277, + "p99": 19091.798381244298 + }, + "tpot_ms": { + "mean": 9.703946782833807, + "p50": 10.335227133570395, + "p90": 10.380124445705588, + "p95": 10.392965908855583, + "p99": 10.40323907937558 + }, + "itl_ms": { + "mean": 11.574165675833974, + "p50": 11.489819007692859, + "p90": 12.426574598066509, + "p95": 12.902599995140916, + "p99": 18.236666286247797 + }, + "gpu_memory": { + "peak_used_mb": 141258.0, + "avg_used_mb": 140860.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_524288_128.jsonl" + }, + { + "name": "c2_i524288_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 14.471764257003088, + "request_throughput": 0.6910007530810116, + "input_token_throughput": 114432.8342136044, + "output_token_throughput": 52.239656932924476, + "total_token_throughput": 114485.07387053731, + "total_input_tokens": 1656045, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 2876.667488797102, + "p50": 1757.1999240026344, + "p90": 7034.463464684086, + "p95": 7604.526347841601, + "p99": 8060.576654367615 + }, + "ttft_ms": { + "mean": 1445.0251086003846, + "p50": 654.1992900019977, + "p90": 2486.3825416890886, + "p95": 4695.872217346909, + "p99": 6463.463957873174 + }, + "tpot_ms": { + "mean": 15.76570859944093, + "p50": 10.899829516808131, + "p90": 17.570392465287224, + "p95": 44.14282767919084, + "p99": 65.40077585031383 + }, + "itl_ms": { + "mean": 21.657978081799207, + "p50": 11.455839994596317, + "p90": 14.507261977996677, + "p95": 17.985764978220686, + "p99": 379.58602359867643 + }, + "gpu_memory": { + "peak_used_mb": 141258.0, + "avg_used_mb": 140860.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_2_524288_128.jsonl" + }, + { + "name": "c1_i1048576_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1048576, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 78.14055735501461, + "request_throughput": 0.06398725795214882, + "input_token_throughput": 19836.894596971106, + "output_token_throughput": 4.850234152772881, + "total_token_throughput": 19841.744831123877, + "total_input_tokens": 1550066, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 15626.558340591146, + "p50": 2301.4842919947114, + "p90": 40110.858696582734, + "p95": 46632.29924277984, + "p99": 51849.45167973754 + }, + "ttft_ms": { + "mean": 14906.361801398452, + "p50": 1038.3560300106183, + "p90": 39440.17306038296, + "p95": 46081.6615871794, + "p99": 51394.85240861657 + }, + "tpot_ms": { + "mean": 8.948563889984472, + "p50": 10.10617233331169, + "p90": 10.48167369132643, + "p95": 10.495872739994212, + "p99": 10.507231978928438 + }, + "itl_ms": { + "mean": 11.577846054635325, + "p50": 11.471398989669979, + "p90": 12.388931994792074, + "p95": 12.913330006995238, + "p99": 16.223136297776342 + }, + "gpu_memory": { + "peak_used_mb": 141258.0, + "avg_used_mb": 140860.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_1_1048576_128.jsonl" + }, + { + "name": "c2_i1048576_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 1048576, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 181.75334272100008, + "request_throughput": 0.055019620823978294, + "input_token_throughput": 17765.334885513093, + "output_token_throughput": 4.1594833342927595, + "total_token_throughput": 17769.494368847387, + "total_input_tokens": 3228909, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 30965.879701598897, + "p50": 32594.995560022653, + "p90": 58127.04405539552, + "p95": 62664.74741968849, + "p99": 66294.9101111229 + }, + "ttft_ms": { + "mean": 24258.31136170309, + "p50": 5467.431873519672, + "p90": 56506.835286997375, + "p95": 61414.69676498527, + "p99": 65340.9859473756 + }, + "tpot_ms": { + "mean": 117.0692356884987, + "p50": 11.282112643952987, + "p90": 461.9875949909769, + "p95": 500.4293399348777, + "p99": 531.1827358899984 + }, + "itl_ms": { + "mean": 106.13154969155116, + "p50": 11.54439800302498, + "p90": 458.76019130228065, + "p95": 734.37676950707, + "p99": 991.9316987271294 + }, + "gpu_memory": { + "peak_used_mb": 141780.0, + "avg_used_mb": 140941.92224880384, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/raw_outputs/vllm_main_2_1048576_128.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/scenarios.tsv b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/scenarios.tsv new file mode 100644 index 0000000..3717ac1 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp2_dp4/scenarios.tsv @@ -0,0 +1,57 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 +Y 1048576 128 1 5 +Y 1048576 128 2 10 diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/report.md b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/report.md new file mode 100644 index 0000000..7695093 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/report.md @@ -0,0 +1,70 @@ +# 8x NVIDIA H200 143GB VLLM Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: VLLM +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i1024_o128 | main | 1 | 1024 | 128 | 3.72 | 5 | 1.34 | 477.80 | 101.85 | 579.65 | 134.63 | 302.85 | 345.83 | 8.17 | 8.32 | 8.35 | 742.89 | 1030.20 | 1055.00 | 138396/143771 MiB (96.3%) | ✅ | +| c128_i1024_o128 | main | 128 | 1024 | 128 | 42.00 | 640 | 15.24 | 7973.13 | 994.43 | 8967.56 | 615.83 | 1659.18 | 1917.69 | 122.83 | 156.63 | 189.20 | 8188.50 | 16737.02 | 18386.47 | 139454/143771 MiB (97.0%) | ⚠️ | +| c1_i1024_o256 | main | 1 | 1024 | 256 | 4.46 | 5 | 1.12 | 399.09 | 113.80 | 512.89 | 76.65 | 88.52 | 90.59 | 8.12 | 8.21 | 8.23 | 889.64 | 1486.72 | 1572.36 | 139454/143771 MiB (97.0%) | ✅ | +| c128_i1024_o256 | main | 128 | 1024 | 256 | 42.40 | 640 | 15.10 | 7897.93 | 1996.42 | 9894.36 | 381.79 | 938.43 | 1171.74 | 60.05 | 74.97 | 96.47 | 8045.59 | 15999.15 | 16952.04 | 139454/143771 MiB (97.0%) | ⚠️ | +| c1_i1024_o512 | main | 1 | 1024 | 512 | 4.58 | 5 | 1.09 | 388.12 | 110.67 | 498.80 | 101.05 | 187.00 | 208.64 | 8.12 | 8.17 | 8.18 | 914.85 | 1486.90 | 1572.17 | 139454/143771 MiB (97.0%) | ✅ | +| c128_i1024_o512 | main | 128 | 1024 | 512 | 54.32 | 640 | 11.78 | 6164.96 | 2986.46 | 9151.42 | 316.84 | 996.03 | 1011.18 | 39.13 | 48.34 | 56.02 | 10050.16 | 20325.81 | 22283.80 | 139456/143771 MiB (97.0%) | ✅ | +| c1_i1024_o1024 | main | 1 | 1024 | 1024 | 12.71 | 5 | 0.39 | 139.93 | 120.49 | 260.41 | 75.65 | 90.47 | 90.97 | 8.11 | 8.19 | 8.21 | 2539.90 | 5592.97 | 5705.34 | 139456/143771 MiB (97.0%) | ✅ | +| c128_i1024_o1024 | main | 128 | 1024 | 1024 | 74.21 | 640 | 8.62 | 4512.43 | 4359.31 | 8871.74 | 224.26 | 719.08 | 837.92 | 26.31 | 30.84 | 33.28 | 13282.76 | 25759.40 | 27367.26 | 139456/143771 MiB (97.0%) | ✅ | +| c1_i1024_o2048 | main | 1 | 1024 | 2048 | 45.78 | 5 | 0.11 | 38.84 | 122.91 | 161.75 | 77.63 | 84.75 | 84.77 | 8.08 | 8.09 | 8.09 | 9154.40 | 13865.78 | 13978.61 | 139456/143771 MiB (97.0%) | ✅ | +| c128_i1024_o2048 | main | 128 | 1024 | 2048 | 132.42 | 640 | 4.83 | 2528.71 | 4909.69 | 7438.40 | 249.17 | 979.55 | 1002.42 | 22.76 | 24.25 | 25.28 | 23085.59 | 43805.99 | 47303.18 | 139456/143771 MiB (97.0%) | ✅ | +| c1_i1024_o4096 | main | 1 | 1024 | 4096 | 95.72 | 5 | 0.05 | 18.57 | 122.97 | 141.55 | 77.88 | 82.10 | 82.14 | 8.09 | 8.12 | 8.13 | 19143.13 | 29602.89 | 30482.21 | 139456/143771 MiB (97.0%) | ✅ | +| c128_i1024_o4096 | main | 128 | 1024 | 4096 | 257.05 | 640 | 2.49 | 1302.66 | 5110.60 | 6413.26 | 197.57 | 737.03 | 754.71 | 21.93 | 23.01 | 23.26 | 44808.43 | 84504.69 | 90427.39 | 139456/143771 MiB (97.0%) | ✅ | +| c1_i4096_o128 | main | 1 | 4096 | 128 | 3.47 | 5 | 1.44 | 2872.58 | 109.20 | 2981.78 | 83.99 | 96.00 | 96.62 | 8.16 | 8.23 | 8.24 | 692.79 | 1028.22 | 1054.23 | 139456/143771 MiB (97.0%) | ✅ | +| c64_i4096_o128 | main | 64 | 4096 | 128 | 30.68 | 320 | 10.43 | 22020.97 | 682.69 | 22703.66 | 584.11 | 1904.20 | 2262.19 | 88.50 | 123.52 | 173.68 | 5949.69 | 11822.35 | 12463.74 | 139456/143771 MiB (97.0%) | ⚠️ | +| c1_i4096_o256 | main | 1 | 4096 | 256 | 4.51 | 5 | 1.11 | 2209.73 | 112.37 | 2322.10 | 83.55 | 91.50 | 91.90 | 8.16 | 8.23 | 8.24 | 900.98 | 1513.84 | 1604.71 | 139456/143771 MiB (97.0%) | ✅ | +| c64_i4096_o256 | main | 64 | 4096 | 256 | 31.29 | 320 | 10.23 | 21590.10 | 1323.77 | 22913.87 | 346.30 | 1198.05 | 1234.72 | 45.94 | 59.77 | 82.58 | 5923.11 | 11845.66 | 13085.09 | 139456/143771 MiB (97.0%) | ✅ | +| c1_i4096_o512 | main | 1 | 4096 | 512 | 4.55 | 5 | 1.10 | 2191.28 | 111.43 | 2302.71 | 91.00 | 104.51 | 107.11 | 8.16 | 8.24 | 8.25 | 908.65 | 1527.91 | 1621.06 | 139456/143771 MiB (97.0%) | ✅ | +| c64_i4096_o512 | main | 64 | 4096 | 512 | 39.52 | 320 | 8.10 | 17097.00 | 2143.08 | 19240.08 | 277.89 | 946.13 | 974.08 | 26.25 | 32.55 | 35.60 | 7228.62 | 14145.53 | 15173.06 | 139456/143771 MiB (97.0%) | ✅ | +| c1_i4096_o1024 | main | 1 | 4096 | 1024 | 12.82 | 5 | 0.39 | 777.58 | 119.41 | 896.98 | 82.49 | 92.40 | 93.46 | 8.15 | 8.23 | 8.24 | 2563.02 | 5648.98 | 5771.30 | 139456/143771 MiB (97.0%) | ✅ | +| c64_i4096_o1024 | main | 64 | 4096 | 1024 | 58.73 | 320 | 5.45 | 11503.60 | 2915.21 | 14418.81 | 261.52 | 1023.51 | 1212.54 | 19.18 | 21.78 | 24.49 | 10436.37 | 20083.65 | 21920.34 | 139824/143771 MiB (97.3%) | ✅ | +| c1_i4096_o2048 | main | 1 | 4096 | 2048 | 46.18 | 5 | 0.11 | 215.88 | 121.84 | 337.72 | 89.17 | 105.61 | 108.34 | 8.13 | 8.16 | 8.16 | 9235.07 | 14025.34 | 14152.06 | 139824/143771 MiB (97.3%) | ✅ | +| c64_i4096_o2048 | main | 64 | 4096 | 2048 | 105.18 | 320 | 3.04 | 6423.33 | 3165.95 | 9589.28 | 219.53 | 856.07 | 869.36 | 17.64 | 18.77 | 20.69 | 18415.93 | 35312.14 | 37188.62 | 139824/143771 MiB (97.3%) | ✅ | +| c1_i4096_o4096 | main | 1 | 4096 | 4096 | 96.28 | 5 | 0.05 | 103.55 | 122.26 | 225.82 | 92.47 | 110.67 | 114.17 | 8.14 | 8.17 | 8.17 | 19254.34 | 29794.95 | 30710.72 | 139824/143771 MiB (97.3%) | ✅ | +| c64_i4096_o4096 | main | 64 | 4096 | 4096 | 196.97 | 320 | 1.62 | 3430.14 | 3323.06 | 6753.20 | 191.83 | 727.11 | 753.66 | 16.53 | 17.22 | 17.64 | 33830.38 | 65711.11 | 68039.71 | 139824/143771 MiB (97.3%) | ✅ | +| c1_i16384_o128 | main | 1 | 16384 | 128 | 4.39 | 5 | 1.14 | 9724.92 | 86.24 | 9811.16 | 272.22 | 512.76 | 518.98 | 8.11 | 8.23 | 8.25 | 877.50 | 1371.53 | 1441.62 | 139824/143771 MiB (97.3%) | ✅ | +| c32_i16384_o128 | main | 32 | 16384 | 128 | 31.01 | 160 | 5.16 | 41639.17 | 340.76 | 41979.93 | 908.42 | 3479.51 | 3760.82 | 85.23 | 160.17 | 235.33 | 6030.16 | 11373.02 | 12999.09 | 139824/143771 MiB (97.3%) | ❌ | +| c1_i16384_o256 | main | 1 | 16384 | 256 | 5.10 | 5 | 0.98 | 8380.37 | 99.42 | 8479.79 | 203.25 | 407.06 | 440.10 | 8.14 | 8.23 | 8.24 | 1018.57 | 1804.68 | 1937.46 | 139824/143771 MiB (97.3%) | ✅ | +| c32_i16384_o256 | main | 32 | 16384 | 256 | 29.56 | 160 | 5.41 | 43677.33 | 751.48 | 44428.80 | 597.59 | 1701.48 | 2572.97 | 38.88 | 55.35 | 126.86 | 5607.62 | 10467.65 | 11645.26 | 140338/143771 MiB (97.6%) | ✅ | +| c1_i16384_o512 | main | 1 | 16384 | 512 | 4.66 | 5 | 1.07 | 9162.19 | 108.69 | 9270.88 | 114.10 | 155.92 | 157.11 | 8.16 | 8.23 | 8.25 | 931.42 | 1574.32 | 1661.52 | 140338/143771 MiB (97.6%) | ✅ | +| c32_i16384_o512 | main | 32 | 16384 | 512 | 29.42 | 160 | 5.44 | 43889.80 | 1373.00 | 45262.81 | 313.79 | 965.45 | 1010.01 | 20.31 | 25.61 | 28.12 | 5422.34 | 10895.57 | 12002.66 | 140854/143771 MiB (98.0%) | ✅ | +| c1_i16384_o1024 | main | 1 | 16384 | 1024 | 13.04 | 5 | 0.38 | 3277.50 | 117.41 | 3394.91 | 114.24 | 158.31 | 161.09 | 8.16 | 8.23 | 8.24 | 2606.54 | 5717.71 | 5837.98 | 140854/143771 MiB (98.0%) | ✅ | +| c32_i16384_o1024 | main | 32 | 16384 | 1024 | 49.82 | 160 | 3.21 | 25915.11 | 1704.78 | 27619.89 | 303.13 | 1131.20 | 1660.57 | 17.14 | 21.79 | 37.19 | 9072.32 | 16361.33 | 18824.49 | 140854/143771 MiB (98.0%) | ✅ | +| c1_i16384_o2048 | main | 1 | 16384 | 2048 | 46.44 | 5 | 0.11 | 920.20 | 121.16 | 1041.36 | 117.87 | 155.57 | 156.44 | 8.14 | 8.18 | 8.19 | 9287.27 | 14109.44 | 14233.84 | 140854/143771 MiB (98.0%) | ✅ | +| c32_i16384_o2048 | main | 32 | 16384 | 2048 | 92.42 | 160 | 1.73 | 13970.52 | 1982.72 | 15953.24 | 323.18 | 1143.92 | 2172.49 | 14.60 | 16.68 | 25.49 | 16690.84 | 28308.53 | 30322.94 | 140854/143771 MiB (98.0%) | ✅ | +| c1_i65536_o128 | main | 1 | 65536 | 128 | 8.91 | 5 | 0.56 | 19510.10 | 42.54 | 19552.65 | 1193.16 | 2211.56 | 2223.60 | 7.89 | 8.16 | 8.19 | 1780.28 | 3036.84 | 3092.42 | 140854/143771 MiB (98.0%) | ✅ | +| c8_i65536_o128 | main | 8 | 65536 | 128 | 42.45 | 40 | 0.94 | 34665.79 | 60.49 | 34726.28 | 2187.88 | 5620.08 | 5766.46 | 118.91 | 301.63 | 309.58 | 8428.69 | 16394.59 | 16549.56 | 140854/143771 MiB (98.0%) | ❌ | +| c1_i65536_o256 | main | 1 | 65536 | 256 | 4.96 | 5 | 1.01 | 35030.52 | 102.18 | 35132.70 | 190.89 | 274.65 | 283.52 | 7.97 | 8.23 | 8.25 | 990.97 | 1580.31 | 1647.86 | 140854/143771 MiB (98.0%) | ✅ | +| c8_i65536_o256 | main | 8 | 65536 | 256 | 30.60 | 40 | 1.31 | 48097.46 | 171.79 | 48269.25 | 1187.02 | 3353.05 | 4860.71 | 44.63 | 113.59 | 141.58 | 5938.64 | 14918.88 | 15303.64 | 140854/143771 MiB (98.0%) | ⚠️ | +| c1_i65536_o512 | main | 1 | 65536 | 512 | 5.10 | 5 | 0.98 | 34094.52 | 99.45 | 34193.97 | 221.62 | 326.62 | 330.25 | 7.91 | 8.17 | 8.18 | 1018.20 | 1589.95 | 1655.09 | 140854/143771 MiB (98.0%) | ✅ | +| c8_i65536_o512 | main | 8 | 65536 | 512 | 27.16 | 40 | 1.47 | 54172.15 | 410.24 | 54582.39 | 493.38 | 1809.22 | 2638.39 | 17.90 | 23.23 | 79.99 | 4993.77 | 8917.36 | 9676.38 | 140854/143771 MiB (98.0%) | ✅ | +| c1_i65536_o1024 | main | 1 | 65536 | 1024 | 13.63 | 5 | 0.37 | 12750.53 | 112.31 | 12862.85 | 229.69 | 358.50 | 364.11 | 8.07 | 8.26 | 8.26 | 2724.95 | 5751.59 | 5842.34 | 140854/143771 MiB (98.0%) | ✅ | +| c8_i65536_o1024 | main | 8 | 65536 | 1024 | 41.44 | 40 | 0.97 | 35510.70 | 528.38 | 36039.07 | 576.47 | 2177.55 | 2783.15 | 13.13 | 20.42 | 32.65 | 7344.83 | 12591.94 | 13878.42 | 140854/143771 MiB (98.0%) | ✅ | +| c1_i131072_o128 | main | 1 | 131072 | 128 | 13.24 | 5 | 0.38 | 27974.16 | 28.62 | 28002.78 | 2074.69 | 3498.43 | 3510.98 | 7.74 | 8.18 | 8.22 | 2646.89 | 4297.81 | 4353.06 | 141374/143771 MiB (98.3%) | ⚠️ | +| c4_i131072_o128 | main | 4 | 131072 | 128 | 32.85 | 20 | 0.61 | 44885.93 | 34.68 | 44920.61 | 2993.02 | 7688.92 | 7746.95 | 55.15 | 151.40 | 157.05 | 6305.32 | 12451.58 | 13546.32 | 141374/143771 MiB (98.3%) | ❌ | +| c1_i131072_o256 | main | 1 | 131072 | 256 | 8.19 | 5 | 0.61 | 45250.75 | 61.94 | 45312.69 | 851.06 | 2612.01 | 3033.01 | 7.81 | 8.23 | 8.24 | 1636.29 | 3468.17 | 3829.62 | 141374/143771 MiB (98.3%) | ✅ | +| c4_i131072_o256 | main | 4 | 131072 | 256 | 25.17 | 20 | 0.79 | 58583.41 | 101.20 | 58684.61 | 1578.82 | 6019.33 | 6092.09 | 20.62 | 43.90 | 46.90 | 4833.02 | 10204.22 | 10875.06 | 141374/143771 MiB (98.3%) | ⚠️ | +| c1_i131072_o512 | main | 1 | 131072 | 512 | 5.60 | 5 | 0.89 | 66152.93 | 90.55 | 66243.48 | 333.30 | 555.04 | 573.87 | 7.81 | 8.24 | 8.27 | 1118.43 | 1626.84 | 1649.69 | 141374/143771 MiB (98.3%) | ✅ | +| c4_i131072_o512 | main | 4 | 131072 | 512 | 19.45 | 20 | 1.03 | 75794.92 | 302.02 | 76096.94 | 489.33 | 724.28 | 1360.67 | 9.94 | 13.97 | 16.16 | 3412.52 | 5788.93 | 5861.60 | 141374/143771 MiB (98.3%) | ✅ | +| c1_i262144_o128 | main | 1 | 262144 | 128 | 24.68 | 5 | 0.20 | 30936.55 | 15.35 | 30951.90 | 4393.17 | 8221.07 | 8613.48 | 6.70 | 7.64 | 7.66 | 4935.30 | 9036.75 | 9509.11 | 141374/143771 MiB (98.3%) | ⚠️ | +| c2_i262144_o128 | main | 2 | 262144 | 128 | 30.76 | 10 | 0.33 | 45318.61 | 24.58 | 45343.19 | 3243.14 | 7994.94 | 8377.33 | 25.23 | 70.28 | 72.19 | 5572.85 | 9102.02 | 9106.67 | 141374/143771 MiB (98.3%) | ⚠️ | +| c1_i262144_o256 | main | 1 | 262144 | 256 | 7.02 | 5 | 0.71 | 108820.99 | 72.25 | 108893.24 | 643.55 | 962.12 | 1036.06 | 6.92 | 7.88 | 7.93 | 1401.77 | 2062.13 | 2080.08 | 141374/143771 MiB (98.3%) | ✅ | +| c2_i262144_o256 | main | 2 | 262144 | 256 | 20.99 | 10 | 0.48 | 66416.34 | 66.52 | 66482.86 | 1808.18 | 6399.54 | 6424.61 | 10.75 | 22.96 | 32.38 | 3491.75 | 7972.48 | 8063.51 | 141374/143771 MiB (98.3%) | ⚠️ | +| c1_i524288_o128 | main | 1 | 524288 | 128 | 23.69 | 5 | 0.21 | 43291.93 | 16.00 | 43307.93 | 4207.11 | 14806.51 | 17551.38 | 6.63 | 7.42 | 7.46 | 4736.94 | 15565.54 | 18281.61 | 141374/143771 MiB (98.3%) | ⚠️ | +| c2_i524288_o128 | main | 2 | 524288 | 128 | 15.61 | 10 | 0.64 | 106086.70 | 48.43 | 106135.13 | 1630.83 | 6241.67 | 9313.98 | 44.54 | 213.86 | 348.24 | 3078.79 | 10277.55 | 10806.98 | 141374/143771 MiB (98.3%) | ⚠️ | +| c1_i1048576_o128 | main | 1 | 1048576 | 128 | 77.70 | 5 | 0.06 | 19948.43 | 4.88 | 19953.31 | 15035.42 | 43228.08 | 48170.56 | 6.14 | 7.51 | 7.56 | 15539.21 | 43599.43 | 48471.96 | 141374/143771 MiB (98.3%) | ⚠️ | +| c2_i1048576_o128 | main | 2 | 1048576 | 128 | 165.96 | 10 | 0.06 | 19455.73 | 4.56 | 19460.29 | 19476.11 | 51577.89 | 52987.83 | 277.94 | 766.41 | 858.05 | 31427.72 | 55532.93 | 56546.05 | 141374/143771 MiB (98.3%) | ❌ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/results.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/results.json new file mode 100644 index 0000000..9bf2564 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/results.json @@ -0,0 +1,3500 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp_dp_matrix_tp4_dp2", + "run_id": "20260709-152419_matrix", + "timestamp": "2026-07-09T16:42:56+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "6fce89c", + "git_dirty": "dirty", + "description": "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 4, + "dp": 2, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 4 --no-enable-flashinfer-autotune --port 30030 --data-parallel-size 2 " + }, + "scenarios": [ + { + "name": "c1_i1024_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.721238563011866, + "request_throughput": 1.3436386609820414, + "input_token_throughput": 477.79790784521396, + "output_token_throughput": 101.84781050243875, + "total_token_throughput": 579.6457183476527, + "total_input_tokens": 1778, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 742.8900246100966, + "p50": 648.4752200194634, + "p90": 999.204512819415, + "p95": 1030.2014534187035, + "p99": 1054.9990058981346 + }, + "ttft_ms": { + "mean": 134.62597660836764, + "p50": 80.85439002024941, + "p90": 249.11989700049165, + "p95": 302.84766700351605, + "p99": 345.8298830059357 + }, + "tpot_ms": { + "mean": 8.170476576695364, + "p50": 8.114223049778957, + "p90": 8.283720724587985, + "p95": 8.321795637463547, + "p99": 8.352255567763997 + }, + "itl_ms": { + "mean": 8.13125441450673, + "p50": 8.066946495091543, + "p90": 8.375731605337931, + "p95": 8.54938578850124, + "p99": 10.11150098434882 + }, + "gpu_memory": { + "peak_used_mb": 138396.0, + "avg_used_mb": 138369.77777777778, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1024_128.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 41.99791230499977, + "request_throughput": 15.23885271611011, + "input_token_throughput": 7973.134416020392, + "output_token_throughput": 994.4303825556603, + "total_token_throughput": 8967.564798576053, + "total_input_tokens": 334855, + "total_output_tokens": 41764, + "e2e_ms": { + "mean": 8188.495863493518, + "p50": 7558.16386899096, + "p90": 15424.557712787648, + "p95": 16737.023079938077, + "p99": 18386.473187913944 + }, + "ttft_ms": { + "mean": 615.8316906878099, + "p50": 518.3006184961414, + "p90": 1314.747746993089, + "p95": 1659.175676117593, + "p99": 1917.6879239003756 + }, + "tpot_ms": { + "mean": 122.8253343572637, + "p50": 131.11594395073783, + "p90": 148.25015912092087, + "p95": 156.63260626495077, + "p99": 189.19791299325874 + }, + "itl_ms": { + "mean": 117.89939475868384, + "p50": 124.28834300953895, + "p90": 195.6383813987486, + "p95": 302.2298654948827, + "p99": 327.8783926862525 + }, + "gpu_memory": { + "peak_used_mb": 139454.0, + "avg_used_mb": 139059.94696969696, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_128_1024_128.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.455133901006775, + "request_throughput": 1.1223007234126219, + "input_token_throughput": 399.09013724552835, + "output_token_throughput": 113.80129335403987, + "total_token_throughput": 512.8914305995683, + "total_input_tokens": 1778, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 889.6417721989565, + "p50": 898.1078299984802, + "p90": 1379.6754283888733, + "p95": 1486.7199551837984, + "p99": 1572.3555766197387 + }, + "ttft_ms": { + "mean": 76.65484239696525, + "p50": 76.37333599268459, + "p90": 85.93957639532164, + "p95": 88.52160719688982, + "p99": 90.58723183814436 + }, + "tpot_ms": { + "mean": 8.124743332363352, + "p50": 8.102052595152728, + "p90": 8.18706675223157, + "p95": 8.209751876100608, + "p99": 8.227899975195841 + }, + "itl_ms": { + "mean": 8.097020840662832, + "p50": 8.049545009271242, + "p90": 8.32335549639538, + "p95": 8.41530616307864, + "p99": 8.954169903299773 + }, + "gpu_memory": { + "peak_used_mb": 139454.0, + "avg_used_mb": 139449.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1024_256.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 42.39779286499834, + "request_throughput": 15.095125400462875, + "input_token_throughput": 7897.934712456243, + "output_token_throughput": 1996.4246787449679, + "total_token_throughput": 9894.359391201211, + "total_input_tokens": 334855, + "total_output_tokens": 84644, + "e2e_ms": { + "mean": 8045.5932444916925, + "p50": 7782.692336491891, + "p90": 14681.59487861849, + "p95": 15999.15330719814, + "p99": 16952.04025982588 + }, + "ttft_ms": { + "mean": 381.79331519691004, + "p50": 277.9630430159159, + "p90": 912.2281327960081, + "p95": 938.4280478814617, + "p99": 1171.7404427187287 + }, + "tpot_ms": { + "mean": 60.04958801017436, + "p50": 61.515032658316734, + "p90": 71.85541459446462, + "p95": 74.96990560988317, + "p99": 96.47274696876381 + }, + "itl_ms": { + "mean": 58.417092998213164, + "p50": 24.514464515959844, + "p90": 176.99348521127834, + "p95": 184.3651863237028, + "p99": 194.41190875077154 + }, + "gpu_memory": { + "peak_used_mb": 139454.0, + "avg_used_mb": 139449.9514925373, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_128_1024_256.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.581034971983172, + "request_throughput": 1.0914564133605498, + "input_token_throughput": 388.1219005910115, + "output_token_throughput": 110.67368031475976, + "total_token_throughput": 498.7955809057713, + "total_input_tokens": 1778, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 914.846669806866, + "p50": 900.4292010213248, + "p90": 1380.3070137975737, + "p95": 1486.899900389835, + "p99": 1572.1742096636444 + }, + "ttft_ms": { + "mean": 101.05085760587826, + "p50": 74.2339999997057, + "p90": 159.94626079918817, + "p95": 186.99729139916596, + "p99": 208.63811587914824 + }, + "tpot_ms": { + "mean": 8.122948848832083, + "p50": 8.113513925797806, + "p90": 8.159414096796905, + "p95": 8.171356448323243, + "p99": 8.180910329544313 + }, + "itl_ms": { + "mean": 8.105077812727819, + "p50": 8.047397481277585, + "p90": 8.229623088845983, + "p95": 8.288775602704845, + "p99": 10.199027028575081 + }, + "gpu_memory": { + "peak_used_mb": 139454.0, + "avg_used_mb": 139450.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1024_512.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 54.31586038999376, + "request_throughput": 11.782930352289933, + "input_token_throughput": 6164.958036118821, + "output_token_throughput": 2986.4573411025854, + "total_token_throughput": 9151.415377221407, + "total_input_tokens": 334855, + "total_output_tokens": 162212, + "e2e_ms": { + "mean": 10050.157637585698, + "p50": 9373.689220505184, + "p90": 18887.582974819816, + "p95": 20325.81091970642, + "p99": 22283.797581810795 + }, + "ttft_ms": { + "mean": 316.84298680143, + "p50": 232.73386000073515, + "p90": 830.1055405085208, + "p95": 996.02754231164, + "p99": 1011.1826269913582 + }, + "tpot_ms": { + "mean": 39.128340454466525, + "p50": 40.164581184674624, + "p90": 46.59944953799815, + "p95": 48.344779400383295, + "p99": 56.0174129189656 + }, + "itl_ms": { + "mean": 38.57011740499057, + "p50": 22.13356748688966, + "p90": 63.24286598828621, + "p95": 172.91890101478202, + "p99": 190.09043615224076 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139450.90476190476, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_128_1024_512.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 12.706748941011028, + "request_throughput": 0.39349168093362585, + "input_token_throughput": 139.92564173999736, + "output_token_throughput": 120.48715270187624, + "total_token_throughput": 260.4127944418736, + "total_input_tokens": 1778, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2539.89674160257, + "p50": 1058.6600590031594, + "p90": 5452.508239808958, + "p95": 5592.96783041209, + "p99": 5705.335502894595 + }, + "ttft_ms": { + "mean": 75.64942800672725, + "p50": 78.58753600157797, + "p90": 89.8377982142847, + "p95": 90.46904561109841, + "p99": 90.97404352854937 + }, + "tpot_ms": { + "mean": 8.111521713687166, + "p50": 8.099772917368442, + "p90": 8.170697841724433, + "p95": 8.190082070732515, + "p99": 8.20558945393898 + }, + "itl_ms": { + "mean": 8.074049801421676, + "p50": 8.056858496274799, + "p90": 8.245306002208963, + "p95": 8.300949004478753, + "p99": 8.51296549080871 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139451.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 74.20719110002392, + "request_throughput": 8.62450108288486, + "input_token_throughput": 4512.433297045953, + "output_token_throughput": 4359.30797547592, + "total_token_throughput": 8871.741272521873, + "total_input_tokens": 334855, + "total_output_tokens": 323492, + "e2e_ms": { + "mean": 13282.760876018028, + "p50": 13183.164570495137, + "p90": 24293.26489258092, + "p95": 25759.395216204575, + "p99": 27367.25632587681 + }, + "ttft_ms": { + "mean": 224.26406962686087, + "p50": 111.3297620031517, + "p90": 671.5504883206451, + "p95": 719.0835456101921, + "p99": 837.91613197478 + }, + "tpot_ms": { + "mean": 26.30878799420686, + "p50": 26.38952824925881, + "p90": 28.918943739017084, + "p95": 30.83845577193247, + "p99": 33.282503663321584 + }, + "itl_ms": { + "mean": 25.89962998858941, + "p50": 21.29472600063309, + "p90": 43.20936339790933, + "p95": 46.90017320099287, + "p99": 103.21796319214592 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139451.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_128_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 45.78009264098364, + "request_throughput": 0.10921777811178691, + "input_token_throughput": 38.837841896551424, + "output_token_throughput": 122.91368748700499, + "total_token_throughput": 161.75152938355643, + "total_input_tokens": 1778, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 9154.397641000105, + "p50": 8897.216579993255, + "p90": 13724.741473596077, + "p95": 13865.781023795716, + "p99": 13978.612663955428 + }, + "ttft_ms": { + "mean": 77.6284478022717, + "p50": 79.23299202229828, + "p90": 84.720662800828, + "p95": 84.74754840717651, + "p99": 84.76905689225532 + }, + "tpot_ms": { + "mean": 8.076663309993533, + "p50": 8.075432795813127, + "p90": 8.087456426545188, + "p95": 8.091436403342898, + "p99": 8.094620384781065 + }, + "itl_ms": { + "mean": 8.072506932233855, + "p50": 8.059671483351849, + "p90": 8.231543694273569, + "p95": 8.282849067472853, + "p99": 8.382042522134725 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139451.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 132.4214202449948, + "request_throughput": 4.833054945460686, + "input_token_throughput": 2528.707209003497, + "output_token_throughput": 4909.689072939647, + "total_token_throughput": 7438.3962819431445, + "total_input_tokens": 334855, + "total_output_tokens": 650148, + "e2e_ms": { + "mean": 23085.5917463311, + "p50": 22873.336221993668, + "p90": 40223.7335820857, + "p95": 43805.991935748876, + "p99": 47303.177158660554 + }, + "ttft_ms": { + "mean": 249.1724353542395, + "p50": 102.51995900762267, + "p90": 953.9042934979079, + "p95": 979.5460204622941, + "p99": 1002.4213002805482 + }, + "tpot_ms": { + "mean": 22.764648204889173, + "p50": 23.208953103962482, + "p90": 23.954017794092255, + "p95": 24.252731038794735, + "p99": 25.282301987327564 + }, + "itl_ms": { + "mean": 22.53238205072957, + "p50": 21.02372798253782, + "p90": 26.16267278790473, + "p95": 37.83958349958992, + "p99": 47.333332192502 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139451.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_128_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 95.72129920398584, + "request_throughput": 0.05223497843823457, + "input_token_throughput": 18.57475833263621, + "output_token_throughput": 122.97158623929182, + "total_token_throughput": 141.54634457192805, + "total_input_tokens": 1778, + "total_output_tokens": 11771, + "e2e_ms": { + "mean": 19143.12961580581, + "p50": 17610.995654016733, + "p90": 28503.73940759455, + "p95": 29602.887815790014, + "p99": 30482.206542346394 + }, + "ttft_ms": { + "mean": 77.87798319477588, + "p50": 81.36516701779328, + "p90": 82.04711719299667, + "p95": 82.10042659775354, + "p99": 82.14307412155904 + }, + "tpot_ms": { + "mean": 8.092311643655734, + "p50": 8.081895106961245, + "p90": 8.123106116408039, + "p95": 8.124891838932284, + "p99": 8.12632041695168 + }, + "itl_ms": { + "mean": 8.101825746048762, + "p50": 8.096112505882047, + "p90": 8.268396501080133, + "p95": 8.321530003740918, + "p99": 8.465466307825409 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139451.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 257.05401054699905, + "request_throughput": 2.489749133414062, + "input_token_throughput": 1302.663978233384, + "output_token_throughput": 5110.599119634458, + "total_token_throughput": 6413.263097867842, + "total_input_tokens": 334855, + "total_output_tokens": 1313700, + "e2e_ms": { + "mean": 44808.432847800716, + "p50": 45304.71561050217, + "p90": 78756.79078810208, + "p95": 84504.68779084622, + "p99": 90427.38911140477 + }, + "ttft_ms": { + "mean": 197.56539933837303, + "p50": 96.52409498812631, + "p90": 601.2236800772371, + "p95": 737.025586295931, + "p99": 754.7124139490188 + }, + "tpot_ms": { + "mean": 21.92814034068577, + "p50": 22.287282601925, + "p90": 22.78568289747275, + "p95": 23.0133071505868, + "p99": 23.259816896142823 + }, + "itl_ms": { + "mean": 21.750477804251904, + "p50": 20.887000500806607, + "p90": 22.717726315022446, + "p95": 26.77762978419194, + "p99": 44.002083983796176 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139452.15410958906, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_128_1024_4096.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.4707513700122945, + "request_throughput": 1.4406102503338603, + "input_token_throughput": 2872.5768391657175, + "output_token_throughput": 109.1982569753066, + "total_token_throughput": 2981.775096141024, + "total_input_tokens": 9970, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 692.7905376069248, + "p50": 674.2977160029113, + "p90": 995.7130086200777, + "p95": 1028.2247638213448, + "p99": 1054.2341679823585 + }, + "ttft_ms": { + "mean": 83.99056080379523, + "p50": 79.96891901711933, + "p90": 95.23673679796048, + "p95": 96.002811397193, + "p99": 96.61567107657902 + }, + "tpot_ms": { + "mean": 8.162352399671141, + "p50": 8.177870149665978, + "p90": 8.21795505574006, + "p95": 8.227887478063511, + "p99": 8.235833415922276 + }, + "itl_ms": { + "mean": 8.138307786181215, + "p50": 8.088838003459387, + "p90": 8.328236304805614, + "p95": 8.429230010369793, + "p99": 9.871305650740377 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139452.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_4096_128.jsonl" + }, + { + "name": "c64_i4096_o128", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 30.681525953987148, + "request_throughput": 10.429728967193535, + "input_token_throughput": 22020.97121939918, + "output_token_throughput": 682.6909467088618, + "total_token_throughput": 22703.662166108043, + "total_input_tokens": 675637, + "total_output_tokens": 20946, + "e2e_ms": { + "mean": 5949.694250859557, + "p50": 5584.752117501921, + "p90": 10855.172719899567, + "p95": 11822.347934574645, + "p99": 12463.743065452436 + }, + "ttft_ms": { + "mean": 584.1092637089787, + "p50": 435.9942669980228, + "p90": 1235.9103825932907, + "p95": 1904.1997952313977, + "p99": 2262.1926730405544 + }, + "tpot_ms": { + "mean": 88.50138026047055, + "p50": 89.95917342028015, + "p90": 108.05219886184639, + "p95": 123.51556064029701, + "p99": 173.68249789637068 + }, + "itl_ms": { + "mean": 83.2676578130938, + "p50": 21.50938200065866, + "p90": 192.7269242965849, + "p95": 199.1564502081019, + "p99": 313.02630993101155 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139452.80454545454, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_64_4096_128.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.511867786990479, + "request_throughput": 1.1081885010941592, + "input_token_throughput": 2209.7278711817535, + "output_token_throughput": 112.37031401094775, + "total_token_throughput": 2322.0981851927013, + "total_input_tokens": 9970, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 900.9787404036615, + "p50": 902.0479850005358, + "p90": 1400.2554600068836, + "p95": 1513.8433510030154, + "p99": 1604.713663799921 + }, + "ttft_ms": { + "mean": 83.55017400463112, + "p50": 81.13631498417817, + "p90": 90.98971001221798, + "p95": 91.49794401600957, + "p99": 91.90453121904284 + }, + "tpot_ms": { + "mean": 8.15765084948505, + "p50": 8.16715459563482, + "p90": 8.217339239593162, + "p95": 8.231538695014331, + "p99": 8.242898259351266 + }, + "itl_ms": { + "mean": 8.141259912392625, + "p50": 8.100810504402034, + "p90": 8.283181901788339, + "p95": 8.344838897755835, + "p99": 8.833542526408564 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139453.75, + "peak_utilization_pct": 98.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_4096_256.jsonl" + }, + { + "name": "c64_i4096_o256", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 31.293840416998137, + "request_throughput": 10.225654497368208, + "input_token_throughput": 21590.09539886989, + "output_token_throughput": 1323.7748850249232, + "total_token_throughput": 22913.870283894812, + "total_input_tokens": 675637, + "total_output_tokens": 41426, + "e2e_ms": { + "mean": 5923.105219631088, + "p50": 5607.908116013277, + "p90": 10917.56653348857, + "p95": 11845.662301342236, + "p99": 13085.091444506545 + }, + "ttft_ms": { + "mean": 346.30415208093837, + "p50": 259.8574580042623, + "p90": 865.4848864011001, + "p95": 1198.0460292965292, + "p99": 1234.7199589200318 + }, + "tpot_ms": { + "mean": 45.93971302723902, + "p50": 45.75125292484674, + "p90": 56.32194467989486, + "p95": 59.768785958604376, + "p99": 82.57894194812978 + }, + "itl_ms": { + "mean": 43.43177547151483, + "p50": 17.583779990673065, + "p90": 175.87184460135177, + "p95": 182.98797760508023, + "p99": 192.98271717270836 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139453.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_64_4096_256.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.549859455000842, + "request_throughput": 1.0989350439175434, + "input_token_throughput": 2191.2764775715814, + "output_token_throughput": 111.4320134532389, + "total_token_throughput": 2302.7084910248204, + "total_input_tokens": 9970, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 908.6494772054721, + "p50": 909.7984280087985, + "p90": 1411.4675548218656, + "p95": 1527.9056514205874, + "p99": 1621.0561286995653 + }, + "ttft_ms": { + "mean": 90.99595120642334, + "p50": 89.84892498119734, + "p90": 101.26915281289257, + "p95": 104.51462041819468, + "p99": 107.11099450243637 + }, + "tpot_ms": { + "mean": 8.15980729287909, + "p50": 8.17331734040326, + "p90": 8.221455233439666, + "p95": 8.236380992096548, + "p99": 8.248321599022052 + }, + "itl_ms": { + "mean": 8.143509780875393, + "p50": 8.110586990369484, + "p90": 8.28320438740775, + "p95": 8.370623656082898, + "p99": 9.090862147568261 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139453.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_4096_512.jsonl" + }, + { + "name": "c64_i4096_o512", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 39.517863471992314, + "request_throughput": 8.09760376410013, + "input_token_throughput": 17097.002232391624, + "output_token_throughput": 2143.081446192625, + "total_token_throughput": 19240.083678584248, + "total_input_tokens": 675637, + "total_output_tokens": 84690, + "e2e_ms": { + "mean": 7228.616324374616, + "p50": 6935.05255949276, + "p90": 13245.782141698875, + "p95": 14145.526426762808, + "p99": 15173.060689594131 + }, + "ttft_ms": { + "mean": 277.88907958192794, + "p50": 141.4275595016079, + "p90": 910.8460375049617, + "p95": 946.125633745396, + "p99": 974.0769699565135 + }, + "tpot_ms": { + "mean": 26.247129394423983, + "p50": 27.070249551602544, + "p90": 30.931282028845214, + "p95": 32.54885823472239, + "p99": 35.59856075350829 + }, + "itl_ms": { + "mean": 26.385256330396412, + "p50": 16.811637993669137, + "p90": 49.47247341624463, + "p95": 60.139739938313, + "p99": 180.50488640205 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139453.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_64_4096_512.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 12.82190531201195, + "request_throughput": 0.38995764500895574, + "input_token_throughput": 777.5755441478577, + "output_token_throughput": 119.40503090174225, + "total_token_throughput": 896.9805750496, + "total_input_tokens": 9970, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2563.0167266062927, + "p50": 1058.7321980274282, + "p90": 5496.073938196059, + "p95": 5648.97837259341, + "p99": 5771.301920111291 + }, + "ttft_ms": { + "mean": 82.49331680126488, + "p50": 78.53085899841972, + "p90": 91.06979420175776, + "p95": 92.39708760287613, + "p99": 93.45892232377082 + }, + "tpot_ms": { + "mean": 8.151794980019885, + "p50": 8.163992704274799, + "p90": 8.211725569792538, + "p95": 8.225680909932814, + "p99": 8.236845182045036 + }, + "itl_ms": { + "mean": 8.127383405628319, + "p50": 8.122238999931142, + "p90": 8.349513504072092, + "p95": 8.392765994358342, + "p99": 8.53409474802902 + }, + "gpu_memory": { + "peak_used_mb": 139456.0, + "avg_used_mb": 139453.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_4096_1024.jsonl" + }, + { + "name": "c64_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 58.73267454802408, + "request_throughput": 5.448415255435795, + "input_token_throughput": 11503.596681052732, + "output_token_throughput": 2915.2086350162685, + "total_token_throughput": 14418.805316069, + "total_input_tokens": 675637, + "total_output_tokens": 171218, + "e2e_ms": { + "mean": 10436.373365484087, + "p50": 10504.825365002034, + "p90": 18472.455321298912, + "p95": 20083.647405948435, + "p99": 21920.335104586557 + }, + "ttft_ms": { + "mean": 261.5205043149217, + "p50": 102.71516301145311, + "p90": 1005.7244442083175, + "p95": 1023.5087016786566, + "p99": 1212.53999641689 + }, + "tpot_ms": { + "mean": 19.178872612605836, + "p50": 19.110644546843904, + "p90": 21.463637178450764, + "p95": 21.78271678596347, + "p99": 24.485239472489173 + }, + "itl_ms": { + "mean": 19.0562145799521, + "p50": 16.3530970166903, + "p90": 20.924012403702374, + "p95": 34.67029542080124, + "p99": 63.48439392721067 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139577.79494382022, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_64_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 46.18259557700367, + "request_throughput": 0.10826589405662852, + "input_token_throughput": 215.88219274891728, + "output_token_throughput": 121.84243717132973, + "total_token_throughput": 337.724629920247, + "total_input_tokens": 9970, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 9235.07033761125, + "p50": 9004.518708999967, + "p90": 13866.931839211611, + "p95": 14025.335175613873, + "p99": 14152.057844735682 + }, + "ttft_ms": { + "mean": 89.16751520009711, + "p50": 90.37135701510124, + "p90": 102.21057139569893, + "p95": 105.6140531960409, + "p99": 108.3368386363145 + }, + "tpot_ms": { + "mean": 8.12746096658243, + "p50": 8.122891957266937, + "p90": 8.162867814509612, + "p95": 8.163429550241974, + "p99": 8.163878938827862 + }, + "itl_ms": { + "mean": 8.133987881721893, + "p50": 8.125334992655553, + "p90": 8.319306685007177, + "p95": 8.373066302738152, + "p99": 8.47865227755392 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139637.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_4096_2048.jsonl" + }, + { + "name": "c64_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 105.18482336500892, + "request_throughput": 3.0422639860272094, + "input_token_throughput": 6423.331602273331, + "output_token_throughput": 3165.9510312091284, + "total_token_throughput": 9589.282633482459, + "total_input_tokens": 675637, + "total_output_tokens": 333010, + "e2e_ms": { + "mean": 18415.92838164952, + "p50": 17792.726314495667, + "p90": 32979.58519420354, + "p95": 35312.143221040606, + "p99": 37188.61529777787 + }, + "ttft_ms": { + "mean": 219.53445320232277, + "p50": 99.16603099554777, + "p90": 832.4278863117797, + "p95": 856.0696429529344, + "p99": 869.3559288178221 + }, + "tpot_ms": { + "mean": 17.642108510335945, + "p50": 17.896604079970523, + "p90": 18.40002850992256, + "p95": 18.76701874358469, + "p99": 20.691679305869712 + }, + "itl_ms": { + "mean": 17.513506270118697, + "p50": 15.979703981429338, + "p90": 17.643520201090723, + "p95": 23.456718999659657, + "p99": 50.41913705761543 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139637.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_64_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 96.27783238200936, + "request_throughput": 0.05193303459680204, + "input_token_throughput": 103.55447098602326, + "output_token_throughput": 122.26075004779136, + "total_token_throughput": 225.81522103381462, + "total_input_tokens": 9970, + "total_output_tokens": 11771, + "e2e_ms": { + "mean": 19254.338539595483, + "p50": 17712.654414004646, + "p90": 28650.246736989357, + "p95": 29794.95201698737, + "p99": 30710.716240985785 + }, + "ttft_ms": { + "mean": 92.46656260220334, + "p50": 92.35269099008292, + "p90": 106.2939728028141, + "p95": 110.66745939897372, + "p99": 114.16624867590144 + }, + "tpot_ms": { + "mean": 8.141002420165762, + "p50": 8.125413391334162, + "p90": 8.168172991052716, + "p95": 8.17006427269898, + "p99": 8.17157729801599 + }, + "itl_ms": { + "mean": 8.142885432604883, + "p50": 8.141148995491676, + "p90": 8.290865502203815, + "p95": 8.334165730047971, + "p99": 8.429930539568886 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139637.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_4096_4096.jsonl" + }, + { + "name": "c64_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 196.97081816900754, + "request_throughput": 1.6246061369630365, + "input_token_throughput": 3430.137551747797, + "output_token_throughput": 3323.0607766393987, + "total_token_throughput": 6753.198328387196, + "total_input_tokens": 675637, + "total_output_tokens": 654546, + "e2e_ms": { + "mean": 33830.37549459041, + "p50": 33209.42994249344, + "p90": 60112.03854111666, + "p95": 65711.10789460945, + "p99": 68039.70623786998 + }, + "ttft_ms": { + "mean": 191.82751641801588, + "p50": 96.30920199560933, + "p90": 569.6680827095406, + "p95": 727.1108964429004, + "p99": 753.6613261778257 + }, + "tpot_ms": { + "mean": 16.52614953470515, + "p50": 16.58643420264918, + "p90": 17.039129384364657, + "p95": 17.220080411641142, + "p99": 17.64209161867442 + }, + "itl_ms": { + "mean": 16.46906162684695, + "p50": 15.793798986123875, + "p90": 17.14870698051527, + "p95": 17.75683920131996, + "p99": 34.682686983142126 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139637.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_64_4096_4096.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.39468751399545, + "request_throughput": 1.1377373212718433, + "input_token_throughput": 9724.923527303208, + "output_token_throughput": 86.24048895240573, + "total_token_throughput": 9811.164016255614, + "total_input_tokens": 42738, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 877.4986200034618, + "p50": 995.6299659970682, + "p90": 1283.916539006168, + "p95": 1371.532126003876, + "p99": 1441.624595602043 + }, + "ttft_ms": { + "mean": 272.21588700194843, + "p50": 189.40226899576373, + "p90": 504.99460959690623, + "p95": 512.7624117885716, + "p99": 518.9766535419039 + }, + "tpot_ms": { + "mean": 8.106169884019737, + "p50": 8.12891391548656, + "p90": 8.21255139165955, + "p95": 8.232200845685679, + "p99": 8.247920408906579 + }, + "itl_ms": { + "mean": 8.112757144796715, + "p50": 8.171425986802205, + "p90": 8.298453403403983, + "p95": 8.353496593190357, + "p99": 8.802264436380932 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139637.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_16384_128.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 31.007318050978938, + "request_throughput": 5.160072204146937, + "input_token_throughput": 41639.17040091244, + "output_token_throughput": 340.75826818135334, + "total_token_throughput": 41979.92866909379, + "total_input_tokens": 1291119, + "total_output_tokens": 10566, + "e2e_ms": { + "mean": 6030.158918974121, + "p50": 5963.966404495295, + "p90": 10334.437299703131, + "p95": 11373.024963619537, + "p99": 12999.094325524346 + }, + "ttft_ms": { + "mean": 908.4206920557335, + "p50": 687.4444764980581, + "p90": 2020.4356724978418, + "p95": 3479.511086911952, + "p99": 3760.819913863379 + }, + "tpot_ms": { + "mean": 85.23109945686367, + "p50": 82.87984177089432, + "p90": 122.94080610110656, + "p95": 160.16552974955013, + "p99": 235.3317044479384 + }, + "itl_ms": { + "mean": 79.13033230404655, + "p50": 14.328335513710044, + "p90": 287.1708374877926, + "p95": 316.8311752669979, + "p99": 376.5704659643234 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139638.05, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_32_16384_128.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.09977360398625, + "request_throughput": 0.9804356797509085, + "input_token_throughput": 8380.372016238865, + "output_token_throughput": 99.41617792674212, + "total_token_throughput": 8479.788194165609, + "total_input_tokens": 42738, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1018.572311796015, + "p50": 1074.4801519904286, + "p90": 1638.7087909970433, + "p95": 1804.6806729980742, + "p99": 1937.4581785988994 + }, + "ttft_ms": { + "mean": 203.24994219699875, + "p50": 159.71414401428774, + "p90": 365.7567747926805, + "p95": 407.05845238990145, + "p99": 440.0997944676783 + }, + "tpot_ms": { + "mean": 8.143973614199677, + "p50": 8.109469493050915, + "p90": 8.210399791161192, + "p95": 8.226191845765866, + "p99": 8.238825489449606 + }, + "itl_ms": { + "mean": 8.13648705786659, + "p50": 8.161870995536447, + "p90": 8.321528002852574, + "p95": 8.370989991817623, + "p99": 8.841439994284883 + }, + "gpu_memory": { + "peak_used_mb": 139824.0, + "avg_used_mb": 139639.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_16384_256.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 29.56039488798706, + "request_throughput": 5.412647584928638, + "input_token_throughput": 43677.32585753423, + "output_token_throughput": 751.4784590725297, + "total_token_throughput": 44428.804316606765, + "total_input_tokens": 1291119, + "total_output_tokens": 22214, + "e2e_ms": { + "mean": 5607.618775876108, + "p50": 5105.33775499789, + "p90": 9879.301131807733, + "p95": 10467.650522004986, + "p99": 11645.257351046312 + }, + "ttft_ms": { + "mean": 597.5871927063054, + "p50": 437.5608239934081, + "p90": 1639.2458354792325, + "p95": 1701.4821891032575, + "p99": 2572.9686887274147 + }, + "tpot_ms": { + "mean": 38.882393224988, + "p50": 38.051979200099595, + "p90": 49.37223125852247, + "p95": 55.3454732937098, + "p99": 126.85517602935364 + }, + "itl_ms": { + "mean": 36.51273482805573, + "p50": 13.292896983330138, + "p90": 76.30884070240428, + "p95": 260.71366565447534, + "p99": 291.7771289261873 + }, + "gpu_memory": { + "peak_used_mb": 140338.0, + "avg_used_mb": 139886.85185185185, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_32_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.664603701996384, + "request_throughput": 1.0719024207479986, + "input_token_throughput": 9162.193131585593, + "output_token_throughput": 108.69090546384706, + "total_token_throughput": 9270.88403704944, + "total_input_tokens": 42738, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 931.4195751969237, + "p50": 938.0052679916844, + "p90": 1465.3151820064522, + "p95": 1574.3161850026809, + "p99": 1661.5169873996638 + }, + "ttft_ms": { + "mean": 114.09855700330809, + "p50": 106.6119369934313, + "p90": 154.4245808152482, + "p95": 155.91656641336158, + "p99": 157.1101548918523 + }, + "tpot_ms": { + "mean": 8.155400557707189, + "p50": 8.150915009786795, + "p90": 8.213660326579516, + "p95": 8.232808513492545, + "p99": 8.248127063022967 + }, + "itl_ms": { + "mean": 8.14015101784037, + "p50": 8.16920849320013, + "p90": 8.31848010129761, + "p95": 8.359751298849005, + "p99": 8.989940869214479 + }, + "gpu_memory": { + "peak_used_mb": 140338.0, + "avg_used_mb": 140085.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_16384_512.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 29.41728782298742, + "request_throughput": 5.438978636058078, + "input_token_throughput": 43889.80411005418, + "output_token_throughput": 1373.0021694399109, + "total_token_throughput": 45262.80627949409, + "total_input_tokens": 1291119, + "total_output_tokens": 40390, + "e2e_ms": { + "mean": 5422.335548431511, + "p50": 4861.034579997067, + "p90": 9935.616184407263, + "p95": 10895.570411083463, + "p99": 12002.661855036857 + }, + "ttft_ms": { + "mean": 313.7887812115878, + "p50": 185.7860879972577, + "p90": 717.2196262981743, + "p95": 965.4450014510074, + "p99": 1010.0096490039141 + }, + "tpot_ms": { + "mean": 20.31214420877035, + "p50": 20.513349666405176, + "p90": 24.077787990479806, + "p95": 25.607989078959847, + "p99": 28.121488541834363 + }, + "itl_ms": { + "mean": 20.385150807390108, + "p50": 12.747994507662952, + "p90": 22.309271997073665, + "p95": 54.75893750553951, + "p99": 216.78622414765437 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140231.28773584904, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_32_16384_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 13.03982701501809, + "request_throughput": 0.38344066943844063, + "input_token_throughput": 3277.497466092015, + "output_token_throughput": 117.40953298205052, + "total_token_throughput": 3394.906999074066, + "total_input_tokens": 42738, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2606.541334395297, + "p50": 1140.9922299790196, + "p90": 5567.364295793232, + "p95": 5717.706294392701, + "p99": 5837.979893272277 + }, + "ttft_ms": { + "mean": 114.23504619742744, + "p50": 99.76904699578881, + "p90": 154.83683619531803, + "p95": 158.30918259453028, + "p99": 161.0870597139001 + }, + "tpot_ms": { + "mean": 8.162550573447826, + "p50": 8.17030822964012, + "p90": 8.216208814917731, + "p95": 8.229407707511978, + "p99": 8.239966821587378 + }, + "itl_ms": { + "mean": 8.165951017700783, + "p50": 8.172732996172272, + "p90": 8.311030003824271, + "p95": 8.360464998986572, + "p99": 8.46713301143609 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 49.821092842001235, + "request_throughput": 3.2114911751817976, + "input_token_throughput": 25915.10796630967, + "output_token_throughput": 1704.7799467055677, + "total_token_throughput": 27619.887913015238, + "total_input_tokens": 1291119, + "total_output_tokens": 84934, + "e2e_ms": { + "mean": 9072.324467187173, + "p50": 9554.756495999754, + "p90": 15422.759670595404, + "p95": 16361.328559013782, + "p99": 18824.492606802087 + }, + "ttft_ms": { + "mean": 303.1298276508096, + "p50": 150.8763884921791, + "p90": 752.9843283031369, + "p95": 1131.2044626596617, + "p99": 1660.5693382397294 + }, + "tpot_ms": { + "mean": 17.136457382381813, + "p50": 16.412214171525452, + "p90": 19.556467973074525, + "p95": 21.79351695247337, + "p99": 37.19118919999149 + }, + "itl_ms": { + "mean": 16.573541087823227, + "p50": 13.859958009561524, + "p90": 15.146375395124782, + "p95": 22.501056210603558, + "p99": 107.31219191686252 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_32_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 46.44400656002108, + "request_throughput": 0.1076565173923645, + "input_token_throughput": 920.2048480629747, + "output_token_throughput": 121.156644673367, + "total_token_throughput": 1041.3614927363417, + "total_input_tokens": 42738, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 9287.265716999536, + "p50": 9034.327834990108, + "p90": 13953.932464198442, + "p95": 14109.439255605685, + "p99": 14233.844688731479 + }, + "ttft_ms": { + "mean": 117.8685202030465, + "p50": 106.28899198491126, + "p90": 154.4862876064144, + "p95": 155.56900080409832, + "p99": 156.43517136224546 + }, + "tpot_ms": { + "mean": 8.139814384597937, + "p50": 8.16596810866496, + "p90": 8.179636834119448, + "p95": 8.183121171989992, + "p99": 8.185908642286426 + }, + "itl_ms": { + "mean": 8.15487424369235, + "p50": 8.157402990036644, + "p90": 8.296986200730316, + "p95": 8.341667961212806, + "p99": 8.466342803440057 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 92.41738028498366, + "request_throughput": 1.731276081475309, + "input_token_throughput": 13970.521518989497, + "output_token_throughput": 1982.7222913585792, + "total_token_throughput": 15953.243810348076, + "total_input_tokens": 1291119, + "total_output_tokens": 183238, + "e2e_ms": { + "mean": 16690.838252105823, + "p50": 17538.859395004692, + "p90": 26505.350084725065, + "p95": 28308.53404565277, + "p99": 30322.94307942211 + }, + "ttft_ms": { + "mean": 323.1806549762041, + "p50": 138.71007499983534, + "p90": 1080.8269529923562, + "p95": 1143.9156597625697, + "p99": 2172.491064503556 + }, + "tpot_ms": { + "mean": 14.598591226013406, + "p50": 14.4070574506273, + "p90": 16.022179939151584, + "p95": 16.681686655123066, + "p99": 25.48573569263988 + }, + "itl_ms": { + "mean": 14.31456483647756, + "p50": 12.639326494536363, + "p90": 14.497518795542419, + "p95": 14.90802480402635, + "p99": 51.936214853776605 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_32_16384_2048.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 8.908717874990543, + "request_throughput": 0.5612479899084589, + "input_token_throughput": 19510.102625197847, + "output_token_throughput": 42.542597635061185, + "total_token_throughput": 19552.64522283291, + "total_input_tokens": 173810, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 1780.2822354016826, + "p50": 2178.572840988636, + "p90": 2967.3524308018386, + "p95": 3036.8367914052214, + "p99": 3092.4242798879277 + }, + "ttft_ms": { + "mean": 1193.1577254086733, + "p50": 1373.9548500161618, + "p90": 2196.5134602040052, + "p95": 2211.5634405985475, + "p99": 2223.6034249141812 + }, + "tpot_ms": { + "mean": 7.894534837551642, + "p50": 7.892211842877898, + "p90": 8.117741047074862, + "p95": 8.158157373933742, + "p99": 8.190490435420847 + }, + "itl_ms": { + "mean": 8.176474356446624, + "p50": 8.269965008366853, + "p90": 8.471901004668325, + "p95": 8.5723778931424, + "p99": 10.3457129985327 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_65536_128.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 42.45026601900463, + "request_throughput": 0.942279136297811, + "input_token_throughput": 34665.789829001056, + "output_token_throughput": 60.49432055031947, + "total_token_throughput": 34726.28414955138, + "total_input_tokens": 1471572, + "total_output_tokens": 2568, + "e2e_ms": { + "mean": 8428.691745326068, + "p50": 8212.495325497002, + "p90": 14741.123111880735, + "p95": 16394.59441000945, + "p99": 16549.55801117263 + }, + "ttft_ms": { + "mean": 2187.8825422718364, + "p50": 2190.3561929939315, + "p90": 3844.8033021006268, + "p95": 5620.080803416204, + "p99": 5766.461412440112 + }, + "tpot_ms": { + "mean": 118.90660609446272, + "p50": 113.07719088074825, + "p90": 202.0349033401, + "p95": 301.6251983424202, + "p99": 309.5849867507786 + }, + "itl_ms": { + "mean": 102.34973051988773, + "p50": 9.896511008264497, + "p90": 317.9538198048249, + "p95": 329.674046713626, + "p99": 393.747671394958 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_8_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.961673725018045, + "request_throughput": 1.007724464990252, + "input_token_throughput": 35030.51785199114, + "output_token_throughput": 102.18326075001156, + "total_token_throughput": 35132.70111274115, + "total_input_tokens": 173810, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 990.9721931966487, + "p50": 1040.3953449858818, + "p90": 1495.8696421876086, + "p95": 1580.3088365821168, + "p99": 1647.8601920977235 + }, + "ttft_ms": { + "mean": 190.88517780182883, + "p50": 220.14280201983638, + "p90": 263.5703456006013, + "p95": 274.6538198029156, + "p99": 283.5205991647672 + }, + "tpot_ms": { + "mean": 7.969151561029765, + "p50": 7.941915901884547, + "p90": 8.202125436961998, + "p95": 8.227581068430037, + "p99": 8.247945573604467 + }, + "itl_ms": { + "mean": 8.180374910022705, + "p50": 8.235591987613589, + "p90": 8.446013601496816, + "p95": 8.517567615490407, + "p99": 9.299252630444247 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_65536_256.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 30.595628153998405, + "request_throughput": 1.307376328365155, + "input_token_throughput": 48097.4599571242, + "output_token_throughput": 171.78924954718136, + "total_token_throughput": 48269.24920667138, + "total_input_tokens": 1471572, + "total_output_tokens": 5256, + "e2e_ms": { + "mean": 5938.643281003169, + "p50": 5732.69273652113, + "p90": 10639.590408897493, + "p95": 14918.883552541956, + "p99": 15303.637388587522 + }, + "ttft_ms": { + "mean": 1187.0219872776943, + "p50": 467.0099939976353, + "p90": 3200.386212713784, + "p95": 3353.049842607287, + "p99": 4860.708494472781 + }, + "tpot_ms": { + "mean": 44.63412787993672, + "p50": 33.272964906794506, + "p90": 100.21680506677325, + "p95": 113.58755296002045, + "p99": 141.58472553635613 + }, + "itl_ms": { + "mean": 38.50554212176001, + "p50": 9.629958993173204, + "p90": 63.88599150523078, + "p95": 306.93651526235044, + "p99": 331.01125874964055 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_8_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.097886697010836, + "request_throughput": 0.9807985734425538, + "input_token_throughput": 34094.520010010056, + "output_token_throughput": 99.45297534707495, + "total_token_throughput": 34193.972985357126, + "total_input_tokens": 173810, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1018.1994763959665, + "p50": 1056.3833659980446, + "p90": 1508.5126165882684, + "p95": 1589.9459927866699, + "p99": 1655.0926937453914 + }, + "ttft_ms": { + "mean": 221.62047139718197, + "p50": 248.06780202197842, + "p90": 322.0881185901817, + "p95": 326.62323578842916, + "p99": 330.2513295470271 + }, + "tpot_ms": { + "mean": 7.9134865962759235, + "p50": 7.924662391922218, + "p90": 8.159459676510565, + "p95": 8.17162573889289, + "p99": 8.181358588798755 + }, + "itl_ms": { + "mean": 8.161189641348338, + "p50": 8.249466991401277, + "p90": 8.46966979734134, + "p95": 8.586531548644416, + "p99": 9.508340466709342 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_65536_512.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 27.16473425901495, + "request_throughput": 1.472497379087208, + "input_token_throughput": 54172.14782845302, + "output_token_throughput": 410.2377698136961, + "total_token_throughput": 54582.385598266716, + "total_input_tokens": 1471572, + "total_output_tokens": 11144, + "e2e_ms": { + "mean": 4993.769744873134, + "p50": 5659.099992990377, + "p90": 7970.951543792035, + "p95": 8917.362221342044, + "p99": 9676.380278440483 + }, + "ttft_ms": { + "mean": 493.3822291997785, + "p50": 310.9949789941311, + "p90": 994.0794685913728, + "p95": 1809.2214257514554, + "p99": 2638.3947925921525 + }, + "tpot_ms": { + "mean": 17.904696785791124, + "p50": 14.133043641911359, + "p90": 21.231970009860433, + "p95": 23.23206808979749, + "p99": 79.99220650661975 + }, + "itl_ms": { + "mean": 16.792353923795673, + "p50": 9.794692494324408, + "p90": 11.13799239101354, + "p95": 16.110704706807127, + "p99": 299.0010918772896 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_8_65536_512.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 13.631586004019482, + "request_throughput": 0.3667951769167339, + "input_token_throughput": 12750.533939979505, + "output_token_throughput": 112.31268317190393, + "total_token_throughput": 12862.846623151408, + "total_input_tokens": 173810, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2724.9468181980774, + "p50": 1315.5071219953243, + "p90": 5638.155575603014, + "p95": 5751.589557807893, + "p99": 5842.3367435717955 + }, + "ttft_ms": { + "mean": 229.68910000054166, + "p50": 234.39669999061152, + "p90": 351.4844470017124, + "p95": 358.4967870032414, + "p99": 364.10665900446475 + }, + "tpot_ms": { + "mean": 8.071901209250624, + "p50": 8.18316470141456, + "p90": 8.256781669216025, + "p95": 8.260151384688811, + "p99": 8.262847157067041 + }, + "itl_ms": { + "mean": 8.213330314667415, + "p50": 8.222839998779818, + "p90": 8.36471018847078, + "p95": 8.409789600409567, + "p99": 8.535900008864699 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 41.44024398599868, + "request_throughput": 0.9652452821830564, + "input_token_throughput": 35510.698259817116, + "output_token_throughput": 528.3752674670051, + "total_token_throughput": 36039.07352728412, + "total_input_tokens": 1471572, + "total_output_tokens": 21896, + "e2e_ms": { + "mean": 7344.833625252068, + "p50": 7517.831302000559, + "p90": 12001.890048698988, + "p95": 12591.942219049084, + "p99": 13878.417216385713 + }, + "ttft_ms": { + "mean": 576.4698123020935, + "p50": 329.346632497618, + "p90": 2113.0596766917733, + "p95": 2177.545073386862, + "p99": 2783.1520693635684 + }, + "tpot_ms": { + "mean": 13.128898573851828, + "p50": 11.72786862060573, + "p90": 16.312076118140663, + "p95": 20.41867396816347, + "p99": 32.65012742865675 + }, + "itl_ms": { + "mean": 12.65048005948215, + "p50": 9.59290299215354, + "p90": 10.794561996590346, + "p95": 11.26046699937433, + "p99": 76.53014999232255 + }, + "gpu_memory": { + "peak_used_mb": 140854.0, + "avg_used_mb": 140343.25, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_8_65536_1024.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 13.241434575989842, + "request_throughput": 0.37760259066387697, + "input_token_throughput": 27974.1592857064, + "output_token_throughput": 28.622276372321874, + "total_token_throughput": 28002.78156207872, + "total_input_tokens": 370418, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 2646.886209206423, + "p50": 3979.2891410179436, + "p90": 4228.74905319768, + "p95": 4297.812612593407, + "p99": 4353.063460109988 + }, + "ttft_ms": { + "mean": 2074.6922916034237, + "p50": 3200.7960340124555, + "p90": 3482.7435716055334, + "p95": 3498.4317138034385, + "p99": 3510.9822275617626 + }, + "tpot_ms": { + "mean": 7.739093047863315, + "p50": 7.695843198159633, + "p90": 8.135016023588832, + "p95": 8.184732786903622, + "p99": 8.224506197555456 + }, + "itl_ms": { + "mean": 8.220462135043283, + "p50": 8.32327950047329, + "p90": 8.480935881379992, + "p95": 8.557302453846205, + "p99": 10.57689555454996 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 140528.13888888888, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_131072_128.jsonl" + }, + { + "name": "c4_i131072_o128", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 32.8477290289884, + "request_throughput": 0.6088700982143951, + "input_token_throughput": 44885.93408387011, + "output_token_throughput": 34.6751520933098, + "total_token_throughput": 44920.60923596342, + "total_input_tokens": 1474401, + "total_output_tokens": 1139, + "e2e_ms": { + "mean": 6305.320444246172, + "p50": 6186.205157995573, + "p90": 10649.124635488264, + "p95": 12451.582126750147, + "p99": 13546.316887757273 + }, + "ttft_ms": { + "mean": 2993.023453996284, + "p50": 2445.912541996222, + "p90": 7026.627190900037, + "p95": 7688.920363315265, + "p99": 7746.948921447038 + }, + "tpot_ms": { + "mean": 55.15329067889916, + "p50": 24.824469108214664, + "p90": 150.7561264443696, + "p95": 151.39788335583992, + "p99": 157.04899968672692 + }, + "itl_ms": { + "mean": 64.00490000482382, + "p50": 8.87769999098964, + "p90": 325.44073020108044, + "p95": 378.21677869651467, + "p99": 447.43990241433494 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 140981.16935483873, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_4_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 8.185897247021785, + "request_throughput": 0.6108065920103155, + "input_token_throughput": 45250.75123985541, + "output_token_throughput": 61.935788429845985, + "total_token_throughput": 45312.687028285254, + "total_input_tokens": 370418, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1636.2850127916317, + "p50": 1331.8980940093752, + "p90": 3016.347304591909, + "p95": 3468.1659967871383, + "p99": 3829.6209505433217 + }, + "ttft_ms": { + "mean": 851.0614429891575, + "p50": 396.08997298637405, + "p90": 2085.7629711856134, + "p95": 2612.0096285827453, + "p99": 3033.0069545004512 + }, + "tpot_ms": { + "mean": 7.807918549771148, + "p50": 7.733951413413233, + "p90": 8.200321834276806, + "p95": 8.22503744215863, + "p99": 8.24480992846409 + }, + "itl_ms": { + "mean": 8.21347870501333, + "p50": 8.264218995464034, + "p90": 8.509880202473141, + "p95": 8.6173327465076, + "p99": 9.081349607731688 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_131072_256.jsonl" + }, + { + "name": "c4_i131072_o256", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 25.167552392988, + "request_throughput": 0.7946740186611175, + "input_token_throughput": 58583.40838939852, + "output_token_throughput": 101.20173627649332, + "total_token_throughput": 58684.61012567501, + "total_input_tokens": 1474401, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 4833.018173197343, + "p50": 5363.060645497171, + "p90": 8616.296054696435, + "p95": 10204.22384394042, + "p99": 10875.063044800772 + }, + "ttft_ms": { + "mean": 1578.8164547033375, + "p50": 567.1622425143141, + "p90": 4755.515111584102, + "p95": 6019.33358459064, + "p99": 6092.089868128824 + }, + "tpot_ms": { + "mean": 20.617913648801018, + "p50": 11.254709681312445, + "p90": 43.49905538605527, + "p95": 43.89723899583028, + "p99": 46.902155262173146 + }, + "itl_ms": { + "mean": 27.24282930725671, + "p50": 8.897086983779445, + "p90": 12.533789791632453, + "p95": 193.9040724013352, + "p99": 383.8965240016113 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_4_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.599419089994626, + "request_throughput": 0.892949772045871, + "input_token_throughput": 66152.9337323375, + "output_token_throughput": 90.54510688545132, + "total_token_throughput": 66243.47883922294, + "total_input_tokens": 370418, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1118.428915401455, + "p50": 1203.2813199912198, + "p90": 1598.2713803998195, + "p95": 1626.8388722033703, + "p99": 1649.6928656462112 + }, + "ttft_ms": { + "mean": 333.30124079366215, + "p50": 421.17294599302113, + "p90": 531.4979945891537, + "p95": 555.0389407901093, + "p99": 573.8716977508739 + }, + "tpot_ms": { + "mean": 7.810111934171644, + "p50": 7.718917504116539, + "p90": 8.2163644195943, + "p95": 8.244790234943565, + "p99": 8.267530887222978 + }, + "itl_ms": { + "mean": 8.195010171184853, + "p50": 8.27857299009338, + "p90": 8.465622790390626, + "p95": 8.511578515754081, + "p99": 9.422437305911433 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_131072_512.jsonl" + }, + { + "name": "c4_i131072_o512", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 19.452503250999143, + "request_throughput": 1.0281453107569964, + "input_token_throughput": 75794.92371627133, + "output_token_throughput": 302.0176850348677, + "total_token_throughput": 76096.94140130619, + "total_input_tokens": 1474401, + "total_output_tokens": 5875, + "e2e_ms": { + "mean": 3412.523278799199, + "p50": 3323.5643379885005, + "p90": 5378.397772402969, + "p95": 5788.928604766261, + "p99": 5861.600530559663 + }, + "ttft_ms": { + "mean": 489.3263924503117, + "p50": 510.96236550074536, + "p90": 651.322402607184, + "p95": 724.2809477567795, + "p99": 1360.671731956826 + }, + "tpot_ms": { + "mean": 9.942585465855233, + "p50": 9.45216913136782, + "p90": 12.93200955462183, + "p95": 13.96891684206874, + "p99": 16.164954653375492 + }, + "itl_ms": { + "mean": 10.494167114358234, + "p50": 8.834993990603834, + "p90": 9.19338600942865, + "p95": 11.618338990956545, + "p99": 66.06157450005425 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_4_131072_512.jsonl" + }, + { + "name": "c1_i262144_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 24.683879674994387, + "request_throughput": 0.202561350396841, + "input_token_throughput": 30936.546849788258, + "output_token_throughput": 15.35415036008055, + "total_token_throughput": 30951.901000148337, + "total_input_tokens": 763634, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 4935.3012414008845, + "p50": 6083.7700930133, + "p90": 8446.296632592566, + "p95": 9036.749655794118, + "p99": 9509.11207435536 + }, + "ttft_ms": { + "mean": 4393.166679597925, + "p50": 6002.486003009835, + "p90": 7730.5542895861445, + "p95": 8221.069923788307, + "p99": 8613.482431150042 + }, + "tpot_ms": { + "mean": 6.699663761331911, + "p50": 7.274658267523988, + "p90": 7.622955134360599, + "p95": 7.641578964257113, + "p99": 7.656478028174324 + }, + "itl_ms": { + "mean": 8.339822855509388, + "p50": 8.389922004425898, + "p90": 8.624103013426065, + "p95": 8.7166765879374, + "p99": 13.3421354938764 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_262144_128.jsonl" + }, + { + "name": "c2_i262144_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 30.757805990986526, + "request_throughput": 0.3251207190438247, + "input_token_throughput": 45318.60953959064, + "output_token_throughput": 24.57912635971315, + "total_token_throughput": 45343.18866595035, + "total_input_tokens": 1393901, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 5572.849400006817, + "p50": 6337.936259515118, + "p90": 9096.200442497502, + "p95": 9102.017221246206, + "p99": 9106.670644245169 + }, + "ttft_ms": { + "mean": 3243.136497796513, + "p50": 1354.019629987306, + "p90": 7516.959703492466, + "p95": 7994.942352740326, + "p99": 8377.328472138615 + }, + "tpot_ms": { + "mean": 25.22720349124221, + "p50": 7.827698137014959, + "p90": 67.88999795797623, + "p95": 70.27790312557251, + "p99": 72.18822725964957 + }, + "itl_ms": { + "mean": 34.92752810191563, + "p50": 8.470294997096062, + "p90": 10.573567188112062, + "p95": 352.75189329404344, + "p99": 490.42820408707513 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_2_262144_128.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 7.0173407410038635, + "request_throughput": 0.712520623486886, + "input_token_throughput": 108820.99475915695, + "output_token_throughput": 72.24959122157024, + "total_token_throughput": 108893.24435037852, + "total_input_tokens": 763634, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1401.7727510014083, + "p50": 1229.8007990175392, + "p90": 2039.681368804304, + "p95": 2062.12660340243, + "p99": 2080.082791080931 + }, + "ttft_ms": { + "mean": 643.5495240089949, + "p50": 577.4397689965554, + "p90": 869.6892130130437, + "p95": 962.1196240128482, + "p99": 1036.063952812692 + }, + "tpot_ms": { + "mean": 6.921365473457675, + "p50": 7.585087851217124, + "p90": 7.831707821027708, + "p95": 7.884401405159936, + "p99": 7.926556272465718 + }, + "itl_ms": { + "mean": 8.331523793378517, + "p50": 8.391458977712318, + "p90": 8.569749590242282, + "p95": 8.623586906469427, + "p99": 12.65661973855457 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 99.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_262144_256.jsonl" + }, + { + "name": "c2_i262144_o256", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 20.987319235980976, + "request_throughput": 0.476478195598028, + "input_token_throughput": 66416.34333222869, + "output_token_throughput": 66.51635610548472, + "total_token_throughput": 66482.85968833417, + "total_input_tokens": 1393901, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 3491.746912599774, + "p50": 1792.6828510098858, + "p90": 7858.686492117704, + "p95": 7972.476840547461, + "p99": 8063.509119291267 + }, + "ttft_ms": { + "mean": 1808.1756318977568, + "p50": 576.1903529928531, + "p90": 6368.217996598105, + "p95": 6399.544915788283, + "p99": 6424.606451140426 + }, + "tpot_ms": { + "mean": 10.74726866873382, + "p50": 8.189858736473353, + "p90": 11.178352464814047, + "p95": 22.95827448958619, + "p99": 32.38221210940396 + }, + "itl_ms": { + "mean": 13.111567167450806, + "p50": 8.418185985647142, + "p90": 8.64076518919319, + "p95": 9.175839160161555, + "p99": 319.3491203873425 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_2_262144_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 23.694438540987903, + "request_throughput": 0.21101998223552473, + "input_token_throughput": 43291.93106751842, + "output_token_throughput": 15.995314653452775, + "total_token_throughput": 43307.92638217187, + "total_input_tokens": 1025778, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 4736.943290999625, + "p50": 1072.3386270110495, + "p90": 12170.458413806047, + "p95": 15565.541827410922, + "p99": 18281.60855829483 + }, + "ttft_ms": { + "mean": 4207.112867000978, + "p50": 589.2991840082686, + "p90": 11375.42073140503, + "p95": 14806.511789211068, + "p99": 17551.384635455906 + }, + "tpot_ms": { + "mean": 6.62686549688025, + "p50": 7.088454843124412, + "p90": 7.382344723550501, + "p95": 7.422827795678168, + "p99": 7.455214253380301 + }, + "itl_ms": { + "mean": 8.436058923582825, + "p50": 8.478927513351664, + "p90": 8.69316780299414, + "p95": 8.776555457734501, + "p99": 15.353498535696419 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_524288_128.jsonl" + }, + { + "name": "c2_i524288_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 15.610297531005926, + "request_throughput": 0.640602780320972, + "input_token_throughput": 106086.7031336644, + "output_token_throughput": 48.42957019226548, + "total_token_throughput": 106135.13270385667, + "total_input_tokens": 1656045, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 3078.7888674007263, + "p50": 1312.8249200090067, + "p90": 9615.761693101376, + "p95": 10277.547583042176, + "p99": 10806.97629499482 + }, + "ttft_ms": { + "mean": 1630.8294385031331, + "p50": 558.4827779966872, + "p90": 2401.2759010103737, + "p95": 6241.666616495049, + "p99": 9313.979188882804 + }, + "tpot_ms": { + "mean": 44.5421275913291, + "p50": 7.944679463915087, + "p90": 45.886444982105864, + "p95": 213.85856155622668, + "p99": 348.236254815524 + }, + "itl_ms": { + "mean": 22.76590438676799, + "p50": 8.449637491139583, + "p90": 10.348231488023885, + "p95": 26.07415175589267, + "p99": 423.9938117956624 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_2_524288_128.jsonl" + }, + { + "name": "c1_i1048576_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1048576, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 77.70366387499962, + "request_throughput": 0.06434703012258731, + "input_token_throughput": 19948.428718799685, + "output_token_throughput": 4.877504883292118, + "total_token_throughput": 19953.306223682976, + "total_input_tokens": 1550066, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 15539.210548391566, + "p50": 5986.594451009296, + "p90": 37508.767122181605, + "p95": 43599.43183357827, + "p99": 48471.96360269561 + }, + "ttft_ms": { + "mean": 15035.420496592997, + "p50": 5905.091491993517, + "p90": 37049.968937598176, + "p95": 43228.075727797106, + "p99": 48170.56115995626 + }, + "tpot_ms": { + "mean": 6.144359419189403, + "p50": 7.069847264610158, + "p90": 7.45164397401612, + "p95": 7.510030491148652, + "p99": 7.556739704854678 + }, + "itl_ms": { + "mean": 8.596305214818218, + "p50": 8.52145001408644, + "p90": 8.81586839677766, + "p95": 8.945846802089362, + "p99": 16.056178179569308 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_1_1048576_128.jsonl" + }, + { + "name": "c2_i1048576_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 1048576, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 165.96181306999642, + "request_throughput": 0.06025482498062598, + "input_token_throughput": 19455.734667336805, + "output_token_throughput": 4.5552647685353245, + "total_token_throughput": 19460.28993210534, + "total_input_tokens": 3228909, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 31427.722522203112, + "p50": 33637.77801199467, + "p90": 54266.53705881035, + "p95": 55532.931616414855, + "p99": 56546.04726249847 + }, + "ttft_ms": { + "mean": 19476.10798960377, + "p50": 9854.220522014657, + "p90": 49815.4517834133, + "p95": 51577.88556921587, + "p99": 52987.83259785792 + }, + "tpot_ms": { + "mean": 277.944045389714, + "p50": 72.84710567608471, + "p90": 651.8447391012355, + "p95": 766.4053257987784, + "p99": 858.0537951568133 + }, + "itl_ms": { + "mean": 212.66139607297154, + "p50": 8.927634509745985, + "p90": 760.9854289854413, + "p95": 891.9243723576071, + "p99": 1024.846997949644 + }, + "gpu_memory": { + "peak_used_mb": 141374.0, + "avg_used_mb": 141124.75, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/raw_outputs/vllm_main_2_1048576_128.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/scenarios.tsv b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/scenarios.tsv new file mode 100644 index 0000000..3717ac1 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp4_dp2/scenarios.tsv @@ -0,0 +1,57 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 +Y 1048576 128 1 5 +Y 1048576 128 2 10 diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/report.md b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/report.md new file mode 100644 index 0000000..36bdfcf --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/report.md @@ -0,0 +1,70 @@ +# 8x NVIDIA H200 143GB VLLM Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: VLLM +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i1024_o128 | main | 1 | 1024 | 128 | 3.11 | 5 | 1.61 | 571.42 | 121.80 | 693.22 | 117.30 | 274.43 | 314.97 | 6.74 | 6.78 | 6.79 | 620.97 | 851.31 | 871.45 | 131900/143771 MiB (91.7%) | ✅ | +| c128_i1024_o128 | main | 128 | 1024 | 128 | 25.39 | 640 | 25.21 | 13188.24 | 1644.87 | 14833.11 | 486.29 | 1647.93 | 2127.36 | 71.51 | 108.98 | 146.42 | 4876.35 | 10467.31 | 11987.25 | 132930/143771 MiB (92.5%) | ⚠️ | +| c1_i1024_o256 | main | 1 | 1024 | 256 | 3.58 | 5 | 1.40 | 497.17 | 141.77 | 638.94 | 40.11 | 44.63 | 45.58 | 6.71 | 6.72 | 6.72 | 713.89 | 1209.48 | 1281.54 | 132930/143771 MiB (92.5%) | ✅ | +| c128_i1024_o256 | main | 128 | 1024 | 256 | 27.46 | 640 | 23.31 | 12194.74 | 3082.56 | 15277.30 | 258.20 | 725.46 | 741.75 | 37.61 | 47.17 | 56.15 | 5081.28 | 9930.41 | 10662.58 | 132930/143771 MiB (92.5%) | ✅ | +| c1_i1024_o512 | main | 1 | 1024 | 512 | 3.69 | 5 | 1.36 | 482.38 | 137.55 | 619.93 | 60.69 | 65.04 | 65.69 | 6.74 | 6.78 | 6.79 | 735.69 | 1233.35 | 1305.74 | 132930/143771 MiB (92.5%) | ✅ | +| c128_i1024_o512 | main | 128 | 1024 | 512 | 37.45 | 640 | 17.09 | 8940.37 | 4330.93 | 13271.30 | 211.37 | 760.28 | 777.02 | 26.29 | 29.37 | 34.23 | 6728.56 | 13132.89 | 14073.81 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i1024_o1024 | main | 1 | 1024 | 1024 | 10.48 | 5 | 0.48 | 169.65 | 146.08 | 315.74 | 48.51 | 60.67 | 60.68 | 6.71 | 6.73 | 6.73 | 2094.65 | 4631.94 | 4727.62 | 132932/143771 MiB (92.5%) | ✅ | +| c128_i1024_o1024 | main | 128 | 1024 | 1024 | 68.42 | 640 | 9.35 | 4893.76 | 4727.69 | 9621.45 | 198.71 | 724.06 | 738.02 | 24.27 | 25.99 | 28.67 | 12260.92 | 23755.36 | 25355.91 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i1024_o2048 | main | 1 | 1024 | 2048 | 37.94 | 5 | 0.13 | 46.87 | 148.32 | 195.19 | 46.74 | 60.19 | 62.81 | 6.71 | 6.72 | 6.72 | 7585.93 | 11492.67 | 11586.89 | 132932/143771 MiB (92.5%) | ✅ | +| c128_i1024_o2048 | main | 128 | 1024 | 2048 | 133.47 | 640 | 4.80 | 2508.84 | 4871.12 | 7379.96 | 198.26 | 746.05 | 761.60 | 23.37 | 24.26 | 24.89 | 23639.36 | 44752.06 | 48201.59 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i1024_o4096 | main | 1 | 1024 | 4096 | 79.46 | 5 | 0.06 | 22.38 | 148.14 | 170.51 | 42.13 | 46.60 | 47.48 | 6.73 | 6.76 | 6.76 | 15890.77 | 24601.03 | 25332.98 | 132932/143771 MiB (92.5%) | ✅ | +| c128_i1024_o4096 | main | 128 | 1024 | 4096 | 275.08 | 640 | 2.33 | 1217.30 | 4775.68 | 5992.98 | 218.44 | 736.09 | 756.63 | 23.99 | 25.92 | 26.62 | 48923.09 | 92762.87 | 97316.79 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i4096_o128 | main | 1 | 4096 | 128 | 3.09 | 5 | 1.62 | 3231.34 | 122.84 | 3354.18 | 113.28 | 148.51 | 155.20 | 6.72 | 6.74 | 6.74 | 615.65 | 898.15 | 918.13 | 132932/143771 MiB (92.5%) | ✅ | +| c64_i4096_o128 | main | 64 | 4096 | 128 | 27.10 | 320 | 11.81 | 24931.54 | 772.92 | 25704.46 | 705.87 | 2978.27 | 3725.28 | 76.32 | 108.07 | 233.29 | 5252.69 | 10435.23 | 11989.51 | 132932/143771 MiB (92.5%) | ⚠️ | +| c1_i4096_o256 | main | 1 | 4096 | 256 | 3.66 | 5 | 1.37 | 2724.98 | 138.57 | 2863.55 | 54.71 | 67.18 | 67.54 | 6.73 | 6.77 | 6.78 | 731.14 | 1243.27 | 1320.30 | 132932/143771 MiB (92.5%) | ✅ | +| c64_i4096_o256 | main | 64 | 4096 | 256 | 16.45 | 320 | 19.45 | 41074.35 | 2518.43 | 43592.78 | 216.47 | 804.28 | 815.33 | 22.25 | 25.88 | 28.53 | 2990.66 | 5802.25 | 6644.63 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i4096_o512 | main | 1 | 4096 | 512 | 3.66 | 5 | 1.37 | 2724.45 | 138.55 | 2863.00 | 53.85 | 64.63 | 65.10 | 6.73 | 6.78 | 6.78 | 730.54 | 1243.77 | 1320.67 | 132932/143771 MiB (92.5%) | ✅ | +| c64_i4096_o512 | main | 64 | 4096 | 512 | 27.32 | 320 | 11.71 | 24731.28 | 3100.03 | 27831.30 | 174.83 | 584.91 | 601.95 | 17.83 | 18.78 | 19.95 | 4860.53 | 9051.41 | 9668.74 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i4096_o1024 | main | 1 | 4096 | 1024 | 10.58 | 5 | 0.47 | 942.72 | 144.76 | 1087.48 | 54.50 | 66.58 | 66.81 | 6.73 | 6.78 | 6.79 | 2113.75 | 4687.47 | 4791.74 | 132932/143771 MiB (92.5%) | ✅ | +| c64_i4096_o1024 | main | 64 | 4096 | 1024 | 52.14 | 320 | 6.14 | 12958.58 | 3283.93 | 16242.51 | 197.14 | 789.79 | 800.08 | 17.06 | 17.95 | 19.33 | 9239.51 | 17164.77 | 17906.82 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i4096_o2048 | main | 1 | 4096 | 2048 | 38.30 | 5 | 0.13 | 260.29 | 146.91 | 407.20 | 55.00 | 67.22 | 67.42 | 6.75 | 6.79 | 6.80 | 7659.17 | 11647.12 | 11755.04 | 132932/143771 MiB (92.5%) | ✅ | +| c64_i4096_o2048 | main | 64 | 4096 | 2048 | 97.69 | 320 | 3.28 | 6916.20 | 3408.88 | 10325.08 | 162.14 | 570.75 | 582.03 | 16.56 | 17.06 | 18.07 | 17265.87 | 32918.48 | 34336.54 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i4096_o4096 | main | 1 | 4096 | 4096 | 79.98 | 5 | 0.06 | 124.66 | 147.18 | 271.85 | 55.41 | 66.70 | 66.85 | 6.77 | 6.80 | 6.80 | 15993.58 | 24761.64 | 25525.53 | 132932/143771 MiB (92.5%) | ✅ | +| c64_i4096_o4096 | main | 64 | 4096 | 4096 | 192.01 | 320 | 1.67 | 3518.79 | 3408.94 | 6927.73 | 163.81 | 576.41 | 599.00 | 16.44 | 16.79 | 16.98 | 33560.76 | 65515.45 | 68183.67 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i16384_o128 | main | 1 | 16384 | 128 | 3.74 | 5 | 1.34 | 11422.24 | 101.29 | 11523.54 | 244.24 | 433.34 | 440.01 | 6.72 | 6.80 | 6.80 | 746.96 | 1184.46 | 1242.36 | 132932/143771 MiB (92.5%) | ✅ | +| c32_i16384_o128 | main | 32 | 16384 | 128 | 29.67 | 160 | 5.39 | 43522.36 | 356.17 | 43878.53 | 996.18 | 3789.36 | 4532.55 | 78.86 | 150.73 | 226.61 | 5789.68 | 10661.97 | 11830.35 | 132932/143771 MiB (92.5%) | ❌ | +| c1_i16384_o256 | main | 1 | 16384 | 256 | 3.82 | 5 | 1.31 | 11173.76 | 132.55 | 11306.32 | 87.44 | 122.35 | 122.83 | 6.73 | 6.76 | 6.76 | 763.51 | 1299.07 | 1372.37 | 132932/143771 MiB (92.5%) | ✅ | +| c32_i16384_o256 | main | 32 | 16384 | 256 | 12.46 | 160 | 12.84 | 103604.79 | 1782.54 | 105387.34 | 200.10 | 636.48 | 644.96 | 14.84 | 16.94 | 20.78 | 2229.45 | 3832.96 | 4262.99 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i16384_o512 | main | 1 | 16384 | 512 | 3.81 | 5 | 1.31 | 11209.67 | 132.98 | 11342.65 | 85.20 | 118.65 | 119.23 | 6.73 | 6.76 | 6.76 | 761.16 | 1293.05 | 1365.21 | 132932/143771 MiB (92.5%) | ✅ | +| c32_i16384_o512 | main | 32 | 16384 | 512 | 20.43 | 160 | 7.83 | 63203.51 | 1977.19 | 65180.70 | 190.69 | 640.86 | 739.06 | 14.04 | 15.70 | 18.53 | 3673.45 | 7038.21 | 7504.82 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i16384_o1024 | main | 1 | 16384 | 1024 | 10.85 | 5 | 0.46 | 3939.68 | 141.13 | 4080.81 | 96.12 | 132.08 | 134.06 | 6.75 | 6.80 | 6.81 | 2168.26 | 4767.81 | 4873.19 | 132932/143771 MiB (92.5%) | ✅ | +| c32_i16384_o1024 | main | 32 | 16384 | 1024 | 40.02 | 160 | 4.00 | 32262.84 | 2122.35 | 34385.19 | 226.08 | 789.94 | 950.40 | 13.38 | 14.23 | 22.02 | 7223.57 | 13034.14 | 13797.93 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i16384_o2048 | main | 1 | 16384 | 2048 | 38.62 | 5 | 0.13 | 1106.58 | 145.70 | 1252.28 | 92.60 | 132.65 | 135.00 | 6.77 | 6.82 | 6.82 | 7722.67 | 11757.03 | 11867.20 | 132932/143771 MiB (92.5%) | ✅ | +| c32_i16384_o2048 | main | 32 | 16384 | 2048 | 82.31 | 160 | 1.94 | 15686.47 | 2226.25 | 17912.73 | 187.67 | 628.02 | 806.24 | 12.88 | 13.39 | 14.85 | 14894.46 | 25291.54 | 26578.49 | 132932/143771 MiB (92.5%) | ✅ | +| c1_i65536_o128 | main | 1 | 65536 | 128 | 5.98 | 5 | 0.84 | 29066.05 | 63.38 | 29129.43 | 697.62 | 1783.81 | 1903.90 | 6.67 | 6.75 | 6.75 | 1194.51 | 2550.76 | 2686.50 | 133260/143771 MiB (92.7%) | ✅ | +| c8_i65536_o128 | main | 8 | 65536 | 128 | 39.65 | 40 | 1.01 | 37118.27 | 64.77 | 37183.05 | 3209.81 | 7445.34 | 8645.53 | 88.42 | 248.75 | 263.80 | 7821.82 | 14289.92 | 16484.17 | 133780/143771 MiB (93.1%) | ❌ | +| c1_i65536_o256 | main | 1 | 65536 | 256 | 4.21 | 5 | 1.19 | 41330.67 | 120.56 | 41451.23 | 161.62 | 254.47 | 256.20 | 6.74 | 6.79 | 6.80 | 839.71 | 1306.35 | 1352.80 | 133780/143771 MiB (93.1%) | ✅ | +| c8_i65536_o256 | main | 8 | 65536 | 256 | 8.64 | 40 | 4.63 | 170252.44 | 608.09 | 170860.53 | 356.04 | 749.11 | 864.77 | 9.14 | 10.21 | 10.30 | 1581.50 | 2865.09 | 2998.37 | 133780/143771 MiB (93.1%) | ✅ | +| c1_i65536_o512 | main | 1 | 65536 | 512 | 4.23 | 5 | 1.18 | 41120.06 | 119.95 | 41240.01 | 184.79 | 288.97 | 290.20 | 6.54 | 6.74 | 6.75 | 843.98 | 1310.36 | 1356.99 | 133780/143771 MiB (93.1%) | ✅ | +| c8_i65536_o512 | main | 8 | 65536 | 512 | 15.79 | 40 | 2.53 | 93186.96 | 705.69 | 93892.65 | 274.50 | 661.39 | 679.12 | 9.10 | 9.63 | 10.39 | 2813.45 | 5054.25 | 5200.71 | 133780/143771 MiB (93.1%) | ✅ | +| c1_i65536_o1024 | main | 1 | 65536 | 1024 | 11.22 | 5 | 0.45 | 15489.07 | 136.44 | 15625.51 | 161.17 | 254.42 | 254.84 | 6.76 | 6.85 | 6.86 | 2242.90 | 4765.81 | 4841.93 | 133780/143771 MiB (93.1%) | ✅ | +| c8_i65536_o1024 | main | 8 | 65536 | 1024 | 29.80 | 40 | 1.34 | 49382.35 | 734.78 | 50117.12 | 281.66 | 671.08 | 707.77 | 8.91 | 9.20 | 10.49 | 5146.09 | 8994.81 | 9476.99 | 133780/143771 MiB (93.1%) | ✅ | +| c1_i131072_o128 | main | 1 | 131072 | 128 | 9.02 | 5 | 0.55 | 41083.10 | 42.03 | 41125.13 | 1332.75 | 3042.40 | 3072.01 | 6.34 | 6.70 | 6.72 | 1801.84 | 3782.08 | 3832.09 | 133780/143771 MiB (93.1%) | ⚠️ | +| c4_i131072_o128 | main | 4 | 131072 | 128 | 24.36 | 20 | 0.82 | 60525.08 | 46.76 | 60571.83 | 2268.91 | 5665.45 | 7563.45 | 46.92 | 130.55 | 135.39 | 4786.17 | 9054.74 | 9263.16 | 134302/143771 MiB (93.4%) | ⚠️ | +| c1_i131072_o256 | main | 1 | 131072 | 256 | 4.85 | 5 | 1.03 | 76336.33 | 104.48 | 76440.81 | 324.28 | 526.13 | 531.79 | 6.38 | 6.74 | 6.75 | 969.07 | 1348.49 | 1365.14 | 134302/143771 MiB (93.4%) | ✅ | +| c4_i131072_o256 | main | 4 | 131072 | 256 | 8.32 | 20 | 2.40 | 177141.68 | 306.01 | 177447.69 | 542.06 | 826.11 | 873.84 | 7.11 | 8.42 | 8.69 | 1532.39 | 2573.00 | 2602.99 | 134302/143771 MiB (93.4%) | ✅ | +| c1_i131072_o512 | main | 1 | 131072 | 512 | 4.87 | 5 | 1.03 | 76048.83 | 104.09 | 76152.92 | 327.91 | 528.58 | 529.52 | 6.38 | 6.74 | 6.75 | 972.74 | 1353.22 | 1365.66 | 134302/143771 MiB (93.4%) | ✅ | +| c4_i131072_o512 | main | 4 | 131072 | 512 | 15.36 | 20 | 1.30 | 95969.05 | 382.40 | 96351.45 | 384.13 | 581.21 | 686.73 | 7.67 | 8.27 | 8.30 | 2674.27 | 3889.94 | 4365.16 | 134302/143771 MiB (93.4%) | ✅ | +| c1_i262144_o128 | main | 1 | 262144 | 128 | 21.96 | 5 | 0.23 | 34775.53 | 17.26 | 34792.78 | 3949.96 | 7547.16 | 7933.13 | 5.31 | 6.25 | 6.26 | 4389.70 | 8216.37 | 8667.71 | 134304/143771 MiB (93.4%) | ⚠️ | +| c2_i262144_o128 | main | 2 | 262144 | 128 | 17.30 | 10 | 0.58 | 80576.29 | 43.70 | 80619.99 | 1811.47 | 6405.29 | 6864.92 | 12.04 | 40.84 | 63.02 | 2911.30 | 7493.52 | 7624.83 | 134304/143771 MiB (93.4%) | ⚠️ | +| c1_i262144_o256 | main | 1 | 262144 | 256 | 5.49 | 5 | 0.91 | 139043.08 | 92.31 | 139135.39 | 483.52 | 741.06 | 802.19 | 5.45 | 6.50 | 6.55 | 1096.55 | 1644.68 | 1664.47 | 134304/143771 MiB (93.4%) | ✅ | +| c2_i262144_o256 | main | 2 | 262144 | 256 | 8.16 | 10 | 1.23 | 170921.30 | 171.18 | 171092.48 | 588.20 | 872.87 | 960.16 | 6.69 | 7.16 | 7.25 | 1528.35 | 2101.10 | 2191.39 | 134304/143771 MiB (93.4%) | ✅ | +| c1_i524288_o128 | main | 1 | 524288 | 128 | 21.47 | 5 | 0.23 | 47769.55 | 17.65 | 47787.20 | 3868.11 | 13982.40 | 16622.77 | 5.18 | 6.10 | 6.17 | 4292.93 | 14596.89 | 17210.26 | 134304/143771 MiB (93.4%) | ⚠️ | +| c2_i524288_o128 | main | 2 | 524288 | 128 | 5.67 | 10 | 1.76 | 292108.80 | 133.35 | 292242.15 | 645.47 | 1167.12 | 1232.75 | 5.64 | 6.90 | 6.94 | 1126.39 | 1682.93 | 1708.86 | 134306/143771 MiB (93.4%) | ✅ | +| c1_i1048576_o128 | main | 1 | 1048576 | 128 | 52.22 | 5 | 0.10 | 29684.37 | 7.26 | 29691.63 | 10048.05 | 38265.96 | 45702.81 | 4.67 | 6.04 | 6.08 | 10442.12 | 38536.13 | 45912.47 | 134310/143771 MiB (93.4%) | ⚠️ | +| c2_i1048576_o128 | main | 2 | 1048576 | 128 | 122.76 | 10 | 0.08 | 26303.08 | 6.16 | 26309.24 | 23067.17 | 59936.57 | 60383.98 | 15.73 | 56.69 | 68.50 | 24549.49 | 63918.14 | 66073.78 | 134312/143771 MiB (93.4%) | ⚠️ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/results.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/results.json new file mode 100644 index 0000000..12fb5cb --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/results.json @@ -0,0 +1,3500 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp_dp_matrix_tp8_dp1", + "run_id": "20260709-152419_matrix", + "timestamp": "2026-07-09T17:51:10+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "6fce89c", + "git_dirty": "dirty", + "description": "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "dp": 1, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 8 --no-enable-flashinfer-autotune --port 30030 " + }, + "scenarios": [ + { + "name": "c1_i1024_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.1115733530023135, + "request_throughput": 1.6069041069449865, + "input_token_throughput": 571.4151004296372, + "output_token_throughput": 121.80333130642998, + "total_token_throughput": 693.2184317360671, + "total_input_tokens": 1778, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 620.9731928014662, + "p50": 539.9669890175574, + "p90": 826.1470417899545, + "p95": 851.3148173864465, + "p99": 871.4490378636401 + }, + "ttft_ms": { + "mean": 117.30040179681964, + "p50": 64.00562298949808, + "p90": 223.75530900317247, + "p95": 274.43198401015246, + "p99": 314.9733240157366 + }, + "tpot_ms": { + "mean": 6.744468233886082, + "p50": 6.731746000107652, + "p90": 6.775324689903452, + "p95": 6.78309076979834, + "p99": 6.78930363371425 + }, + "itl_ms": { + "mean": 6.732948214054745, + "p50": 6.699120509438217, + "p90": 6.850680703064427, + "p95": 6.90187855361728, + "p99": 8.64518792688613 + }, + "gpu_memory": { + "peak_used_mb": 131900.0, + "avg_used_mb": 131900.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1024_128.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 25.39041935800924, + "request_throughput": 25.206357995742053, + "input_token_throughput": 13188.242197912821, + "output_token_throughput": 1644.8723989596424, + "total_token_throughput": 14833.114596872463, + "total_input_tokens": 334855, + "total_output_tokens": 41764, + "e2e_ms": { + "mean": 4876.350990326864, + "p50": 4396.088168999995, + "p90": 8972.985691693613, + "p95": 10467.312990417122, + "p99": 11987.250543742266 + }, + "ttft_ms": { + "mean": 486.2943946186988, + "p50": 297.0849985140376, + "p90": 1172.9323532112176, + "p95": 1647.9308876645518, + "p99": 2127.3599312192528 + }, + "tpot_ms": { + "mean": 71.50531383675863, + "p50": 68.03394203979266, + "p90": 100.23968791809875, + "p95": 108.98483476371113, + "p99": 146.42384668454534 + }, + "itl_ms": { + "mean": 68.34565651707587, + "p50": 23.442019009962678, + "p90": 165.77782639069483, + "p95": 176.94342100294307, + "p99": 287.3323047114536 + }, + "gpu_memory": { + "peak_used_mb": 132930.0, + "avg_used_mb": 132425.22448979592, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_128_1024_128.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.576241757022217, + "request_throughput": 1.3981157706081049, + "input_token_throughput": 497.1699680282421, + "output_token_throughput": 141.76893913966185, + "total_token_throughput": 638.938907167904, + "total_input_tokens": 1778, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 713.893802597886, + "p50": 723.9885129965842, + "p90": 1119.4012374093293, + "p95": 1209.478307212703, + "p99": 1281.539963055402 + }, + "ttft_ms": { + "mean": 40.11499620391987, + "p50": 39.00022400193848, + "p90": 43.437266402179375, + "p95": 44.628948206081986, + "p99": 45.582293649204075 + }, + "tpot_ms": { + "mean": 6.712814833205494, + "p50": 6.714892661785969, + "p90": 6.715674534001772, + "p95": 6.715708891758421, + "p99": 6.715736377963742 + }, + "itl_ms": { + "mean": 6.710483619478992, + "p50": 6.695525997201912, + "p90": 6.968245579628274, + "p95": 7.05675591743784, + "p99": 7.320103088568431 + }, + "gpu_memory": { + "peak_used_mb": 132930.0, + "avg_used_mb": 132930.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1024_256.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 27.458979611023096, + "request_throughput": 23.30749390786099, + "input_token_throughput": 12194.735738307489, + "output_token_throughput": 3082.5617411515404, + "total_token_throughput": 15277.297479459028, + "total_input_tokens": 334855, + "total_output_tokens": 84644, + "e2e_ms": { + "mean": 5081.282830074133, + "p50": 4903.469455995946, + "p90": 9170.879973901903, + "p95": 9930.409122745914, + "p99": 10662.57701508439 + }, + "ttft_ms": { + "mean": 258.1973640362321, + "p50": 173.7810085178353, + "p90": 552.6221598003758, + "p95": 725.4641677965992, + "p99": 741.747970864817 + }, + "tpot_ms": { + "mean": 37.60625974624434, + "p50": 38.062800910555985, + "p90": 44.329389671741716, + "p95": 47.17057905603296, + "p99": 56.149049528066804 + }, + "itl_ms": { + "mean": 36.76333383779741, + "p50": 23.366706998785958, + "p90": 60.13022359111347, + "p95": 96.74666440114366, + "p99": 175.20204528060276 + }, + "gpu_memory": { + "peak_used_mb": 132930.0, + "avg_used_mb": 132930.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_128_1024_256.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.68587921600556, + "request_throughput": 1.3565284446348655, + "input_token_throughput": 482.3815149121582, + "output_token_throughput": 137.55198428597538, + "total_token_throughput": 619.9334991981335, + "total_input_tokens": 1778, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 735.6866433983669, + "p50": 742.1285230084322, + "p90": 1142.8636193915736, + "p95": 1233.3502841880543, + "p99": 1305.7396160252392 + }, + "ttft_ms": { + "mean": 60.687745799077675, + "p50": 61.6830279759597, + "p90": 64.22489199903794, + "p95": 65.0386460009031, + "p99": 65.68964920239523 + }, + "tpot_ms": { + "mean": 6.735674190831488, + "p50": 6.728461088142449, + "p90": 6.764326388062254, + "p95": 6.7760316441065145, + "p99": 6.785395848941923 + }, + "itl_ms": { + "mean": 6.722634358540738, + "p50": 6.6887545108329505, + "p90": 6.855826204991899, + "p95": 6.903062094352208, + "p99": 7.3225617126445295 + }, + "gpu_memory": { + "peak_used_mb": 132930.0, + "avg_used_mb": 132930.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1024_512.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 37.454274895979324, + "request_throughput": 17.08750207492879, + "input_token_throughput": 8940.367980156687, + "output_token_throughput": 4330.93419777867, + "total_token_throughput": 13271.302177935358, + "total_input_tokens": 334855, + "total_output_tokens": 162212, + "e2e_ms": { + "mean": 6728.558528907206, + "p50": 6503.705489492859, + "p90": 12159.59878990543, + "p95": 13132.893477000467, + "p99": 14073.808510748788 + }, + "ttft_ms": { + "mean": 211.3740406251054, + "p50": 116.70142400544137, + "p90": 558.8830624881666, + "p95": 760.2822678250959, + "p99": 777.0182164228754 + }, + "tpot_ms": { + "mean": 26.292669162728608, + "p50": 26.72706325471736, + "p90": 28.302211153833543, + "p95": 29.36632828523293, + "p99": 34.23378473595377 + }, + "itl_ms": { + "mean": 25.846200514553402, + "p50": 22.78870699228719, + "p90": 34.93604898103513, + "p95": 37.07805039011873, + "p99": 89.45705803576855 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132931.16417910447, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_128_1024_512.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 10.480287076003151, + "request_throughput": 0.4770861679398616, + "input_token_throughput": 169.65184131941479, + "output_token_throughput": 146.0837846231856, + "total_token_throughput": 315.7356259426004, + "total_input_tokens": 1778, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2094.6468008100055, + "p50": 850.2973650174681, + "p90": 4512.331283814274, + "p95": 4631.936234416207, + "p99": 4727.620194897754 + }, + "ttft_ms": { + "mean": 48.51190840709023, + "p50": 44.887225987622514, + "p90": 60.66339420503937, + "p95": 60.67136460333131, + "p99": 60.67774092196487 + }, + "tpot_ms": { + "mean": 6.711405225664204, + "p50": 6.707264200667851, + "p90": 6.725284867411174, + "p95": 6.73001691964462, + "p99": 6.733802561431377 + }, + "itl_ms": { + "mean": 6.704089605504419, + "p50": 6.692452501738444, + "p90": 6.84615749923978, + "p95": 6.903180488734506, + "p99": 7.070738494803663 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 68.42493821599055, + "request_throughput": 9.35331498553601, + "input_token_throughput": 4893.756702315094, + "output_token_throughput": 4727.691517657836, + "total_token_throughput": 9621.44821997293, + "total_input_tokens": 334855, + "total_output_tokens": 323492, + "e2e_ms": { + "mean": 12260.91877303138, + "p50": 12370.803522499045, + "p90": 22112.71922251035, + "p95": 23755.357677354186, + "p99": 25355.91061090352 + }, + "ttft_ms": { + "mean": 198.70940640835215, + "p50": 110.31356250168756, + "p90": 590.7794083002955, + "p95": 724.0565157524542, + "p99": 738.0223441249109 + }, + "tpot_ms": { + "mean": 24.27136748121635, + "p50": 24.670942883774895, + "p90": 25.631626756557917, + "p95": 25.98724100797857, + "p99": 28.667557228618534 + }, + "itl_ms": { + "mean": 23.925414834838133, + "p50": 22.627816011663526, + "p90": 31.98559701559134, + "p95": 34.510503988713026, + "p99": 38.06540880468675 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_128_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 37.93773976698867, + "request_throughput": 0.13179488368863568, + "input_token_throughput": 46.86626063967884, + "output_token_throughput": 148.32196210319057, + "total_token_throughput": 195.18822274286944, + "total_input_tokens": 1778, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 7585.934515192639, + "p50": 7374.917035980616, + "p90": 11374.88244700362, + "p95": 11492.666910006665, + "p99": 11586.8944804091 + }, + "ttft_ms": { + "mean": 46.74239640007727, + "p50": 42.968800000380725, + "p90": 56.913516198983416, + "p95": 60.18717059050687, + "p99": 62.806094103725634 + }, + "tpot_ms": { + "mean": 6.707419037359887, + "p50": 6.704420040282551, + "p90": 6.716400699617259, + "p95": 6.718642043996769, + "p99": 6.720435119500378 + }, + "itl_ms": { + "mean": 6.7050363785128875, + "p50": 6.695530493743718, + "p90": 6.846846811822616, + "p95": 6.88849520374788, + "p99": 7.007950085680932 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 133.46989706001477, + "request_throughput": 4.7950887360932315, + "input_token_throughput": 2508.84287300703, + "output_token_throughput": 4871.12086186491, + "total_token_throughput": 7379.96373487194, + "total_input_tokens": 334855, + "total_output_tokens": 650148, + "e2e_ms": { + "mean": 23639.356331531737, + "p50": 23619.91920748551, + "p90": 41096.837318109465, + "p95": 44752.062044409104, + "p99": 48201.59040482453 + }, + "ttft_ms": { + "mean": 198.2606400615623, + "p50": 102.33299050014466, + "p90": 570.831106393598, + "p95": 746.0507048497675, + "p99": 761.6026770081954 + }, + "tpot_ms": { + "mean": 23.365234939241812, + "p50": 23.89083861284945, + "p90": 24.153373456672526, + "p95": 24.25965965126051, + "p99": 24.887882278820108 + }, + "itl_ms": { + "mean": 23.115038657840238, + "p50": 22.805721484473906, + "p90": 25.43376050016377, + "p95": 32.6996164949378, + "p99": 36.15239749342436 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_128_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 79.46116249699844, + "request_throughput": 0.0629238214352687, + "input_token_throughput": 22.375710902381552, + "output_token_throughput": 148.13526042290957, + "total_token_throughput": 170.51097132529114, + "total_input_tokens": 1778, + "total_output_tokens": 11771, + "e2e_ms": { + "mean": 15890.771754994057, + "p50": 14598.433314997237, + "p90": 23686.08512879, + "p95": 24601.027589384466, + "p99": 25332.98155786004 + }, + "ttft_ms": { + "mean": 42.12920940481126, + "p50": 41.481438995106146, + "p90": 45.4949876118917, + "p95": 46.59649881650694, + "p99": 47.477707780199125 + }, + "tpot_ms": { + "mean": 6.725161891015255, + "p50": 6.712620497455561, + "p90": 6.755732904352653, + "p95": 6.7565145338363335, + "p99": 6.757139837423277 + }, + "itl_ms": { + "mean": 6.734912228627602, + "p50": 6.72975298948586, + "p90": 6.884617512696423, + "p95": 6.931063246156555, + "p99": 7.00651356310118 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 275.08098518897896, + "request_throughput": 2.3265875667862828, + "input_token_throughput": 1217.2960619940948, + "output_token_throughput": 4775.684510136155, + "total_token_throughput": 5992.98057213025, + "total_input_tokens": 334855, + "total_output_tokens": 1313700, + "e2e_ms": { + "mean": 48923.094904336176, + "p50": 49395.30037200893, + "p90": 86516.55968841516, + "p95": 92762.86578539002, + "p99": 97316.79100864421 + }, + "ttft_ms": { + "mean": 218.43703098265905, + "p50": 147.93294100672938, + "p90": 567.803863005247, + "p95": 736.089210139471, + "p99": 756.6278313548537 + }, + "tpot_ms": { + "mean": 23.993136593347142, + "p50": 24.11520213519814, + "p90": 25.682799309682903, + "p95": 25.924939912533848, + "p99": 26.623839328782584 + }, + "itl_ms": { + "mean": 23.753848993227077, + "p50": 22.822784987511113, + "p90": 24.192147998837754, + "p95": 27.596254809759543, + "p99": 86.22891287785025 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_128_1024_4096.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.0854074439848773, + "request_throughput": 1.6205315151318818, + "input_token_throughput": 3231.3398411729722, + "output_token_throughput": 122.83628884699664, + "total_token_throughput": 3354.176130019969, + "total_input_tokens": 9970, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 615.6534888024908, + "p50": 561.0614480101503, + "p90": 873.1742330011912, + "p95": 898.1507350050379, + "p99": 918.1319366081152 + }, + "ttft_ms": { + "mean": 113.27877099392936, + "p50": 114.13994699250907, + "p90": 140.1475009857677, + "p95": 148.5101829865016, + "p99": 155.20032858708873 + }, + "tpot_ms": { + "mean": 6.7182663676562875, + "p50": 6.7081030413473774, + "p90": 6.739791897229386, + "p95": 6.740897068400767, + "p99": 6.741781205337871 + }, + "itl_ms": { + "mean": 6.715573219334667, + "p50": 6.7164640058763325, + "p90": 6.88778919284232, + "p95": 6.937314548122231, + "p99": 7.244373657449614 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_4096_128.jsonl" + }, + { + "name": "c64_i4096_o128", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 27.099694294010988, + "request_throughput": 11.80825128609365, + "input_token_throughput": 24931.535856820174, + "output_token_throughput": 772.9238482453675, + "total_token_throughput": 25704.459705065543, + "total_input_tokens": 675637, + "total_output_tokens": 20946, + "e2e_ms": { + "mean": 5252.685820971146, + "p50": 4714.832158992067, + "p90": 9597.868900399773, + "p95": 10435.22773818986, + "p99": 11989.513149040287 + }, + "ttft_ms": { + "mean": 705.8739211091961, + "p50": 366.5684799925657, + "p90": 2010.9284130943706, + "p95": 2978.274026054715, + "p99": 3725.2794846516917 + }, + "tpot_ms": { + "mean": 76.32340626525998, + "p50": 76.2943883227518, + "p90": 96.5839731860407, + "p95": 108.06964486550896, + "p99": 233.29262652507254 + }, + "itl_ms": { + "mean": 70.56425622449346, + "p50": 16.781184007413685, + "p90": 198.48385320510718, + "p95": 248.3880425861571, + "p99": 256.2605793023249 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_64_4096_128.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.658745035994798, + "request_throughput": 1.366588803212553, + "input_token_throughput": 2724.9780736058306, + "output_token_throughput": 138.57210464575286, + "total_token_throughput": 2863.5501782515835, + "total_input_tokens": 9970, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 731.1363978078589, + "p50": 731.1096460034605, + "p90": 1146.9773760065436, + "p95": 1243.268521002028, + "p99": 1320.3014369984155 + }, + "ttft_ms": { + "mean": 54.709906002972275, + "p50": 47.65253601362929, + "p90": 66.72574519761838, + "p95": 67.17990359757096, + "p99": 67.54323031753302 + }, + "tpot_ms": { + "mean": 6.727269818785986, + "p50": 6.707462650956586, + "p90": 6.764210822905439, + "p95": 6.77092561357972, + "p99": 6.776297446119144 + }, + "itl_ms": { + "mean": 6.737199599670981, + "p50": 6.741824996424839, + "p90": 6.881189302657731, + "p95": 6.924558590981178, + "p99": 7.134522728447337 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_4096_256.jsonl" + }, + { + "name": "c64_i4096_o256", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 16.449123202008195, + "request_throughput": 19.453924447531204, + "input_token_throughput": 41074.3473498645, + "output_token_throughput": 2518.4321067607116, + "total_token_throughput": 43592.77945662521, + "total_input_tokens": 675637, + "total_output_tokens": 41426, + "e2e_ms": { + "mean": 2990.6639760214603, + "p50": 2892.804385002819, + "p90": 5216.253019598664, + "p95": 5802.251851401526, + "p99": 6644.63413677091 + }, + "ttft_ms": { + "mean": 216.46908835318754, + "p50": 120.06344750989228, + "p90": 605.4238382988842, + "p95": 804.2780677453267, + "p99": 815.3280701686163 + }, + "tpot_ms": { + "mean": 22.251804362551017, + "p50": 21.73764368671982, + "p90": 24.98197791283019, + "p95": 25.87954078425632, + "p99": 28.534352022444544 + }, + "itl_ms": { + "mean": 21.710834613935567, + "p50": 16.339479014277458, + "p90": 35.72288859868423, + "p95": 47.24770099273881, + "p99": 85.45168400160037 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_64_4096_256.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.6594501009967644, + "request_throughput": 1.3663255030142631, + "input_token_throughput": 2724.4530530104407, + "output_token_throughput": 138.54540600564627, + "total_token_throughput": 2862.998459016087, + "total_input_tokens": 9970, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 730.5364638043102, + "p50": 730.989553994732, + "p90": 1147.649170004297, + "p95": 1243.7729350000152, + "p99": 1320.6719469965901 + }, + "ttft_ms": { + "mean": 53.851668402785435, + "p50": 47.597177006537095, + "p90": 64.02949960320257, + "p95": 64.626579801552, + "p99": 65.10424396023154 + }, + "tpot_ms": { + "mean": 6.728677737284085, + "p50": 6.708104942236404, + "p90": 6.770449891860923, + "p95": 6.7753128182598035, + "p99": 6.779203159378907 + }, + "itl_ms": { + "mean": 6.73937784067023, + "p50": 6.737591014825739, + "p90": 6.900259788380936, + "p95": 6.942523361067288, + "p99": 7.090970962017309 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_4096_512.jsonl" + }, + { + "name": "c64_i4096_o512", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 27.31912976797321, + "request_throughput": 11.713403857217397, + "input_token_throughput": 24731.27825587122, + "output_token_throughput": 3100.0255395866916, + "total_token_throughput": 27831.303795457912, + "total_input_tokens": 675637, + "total_output_tokens": 84690, + "e2e_ms": { + "mean": 4860.531406028804, + "p50": 4964.397494011791, + "p90": 8671.064496890176, + "p95": 9051.41342166462, + "p99": 9668.73572528013 + }, + "ttft_ms": { + "mean": 174.8319179313512, + "p50": 100.47747699718457, + "p90": 527.0181634987239, + "p95": 584.906001255149, + "p99": 601.9537152591511 + }, + "tpot_ms": { + "mean": 17.82869827666415, + "p50": 18.254847125018177, + "p90": 18.573381245585715, + "p95": 18.777003847560263, + "p99": 19.951478950302665 + }, + "itl_ms": { + "mean": 17.790459763842904, + "p50": 16.201541511691175, + "p90": 27.450341390795074, + "p95": 31.0013980633812, + "p99": 36.46449689462316 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_64_4096_512.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 10.57582659801119, + "request_throughput": 0.4727762840722315, + "input_token_throughput": 942.7159104400296, + "output_token_throughput": 144.76409818291728, + "total_token_throughput": 1087.4800086229468, + "total_input_tokens": 9970, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2113.7520164076705, + "p50": 858.8050559919793, + "p90": 4557.140987808816, + "p95": 4687.4714534089435, + "p99": 4791.735825889045 + }, + "ttft_ms": { + "mean": 54.499105404829606, + "p50": 47.3600749974139, + "p90": 66.29754161695018, + "p95": 66.58417182043195, + "p99": 66.81347598321736 + }, + "tpot_ms": { + "mean": 6.734874706680725, + "p50": 6.722214550245553, + "p90": 6.772506061471742, + "p95": 6.78079893645199, + "p99": 6.787433236436187 + }, + "itl_ms": { + "mean": 6.747072798838939, + "p50": 6.746081984601915, + "p90": 6.896372506162152, + "p95": 6.942834756046068, + "p99": 7.040538992441725 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_4096_1024.jsonl" + }, + { + "name": "c64_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 52.138183050992666, + "request_throughput": 6.137536470863027, + "input_token_throughput": 12958.583526764009, + "output_token_throughput": 3283.927248338205, + "total_token_throughput": 16242.510775102213, + "total_input_tokens": 675637, + "total_output_tokens": 171218, + "e2e_ms": { + "mean": 9239.511910518468, + "p50": 9612.64459049562, + "p90": 16225.299288914539, + "p95": 17164.766070038608, + "p99": 17906.819723816297 + }, + "ttft_ms": { + "mean": 197.14426652762995, + "p50": 96.6091855079867, + "p90": 756.5386313915951, + "p95": 789.7942044000956, + "p99": 800.0755239935825 + }, + "tpot_ms": { + "mean": 17.056337422967584, + "p50": 17.351856994488298, + "p90": 17.638552956453005, + "p95": 17.951327604176846, + "p99": 19.325315261683347 + }, + "itl_ms": { + "mean": 16.95405888753211, + "p50": 16.254876492894255, + "p90": 19.355191584327265, + "p95": 27.920874397386786, + "p99": 34.48984432354336 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_64_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 38.30321971999365, + "request_throughput": 0.1305373291475568, + "input_token_throughput": 260.2914343202283, + "output_token_throughput": 146.90671022266045, + "total_token_throughput": 407.1981445428887, + "total_input_tokens": 9970, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 7659.171856596367, + "p50": 7501.7676659917925, + "p90": 11512.226327991812, + "p95": 11647.121157991933, + "p99": 11755.03702199203 + }, + "ttft_ms": { + "mean": 55.00147619750351, + "p50": 47.79739800142124, + "p90": 66.96408639545552, + "p95": 67.21895219525322, + "p99": 67.42284483509138 + }, + "tpot_ms": { + "mean": 6.749552102034963, + "p50": 6.751980732594644, + "p90": 6.793154116167625, + "p95": 6.794435671481723, + "p99": 6.795460915733 + }, + "itl_ms": { + "mean": 6.762827909467697, + "p50": 6.759966490790248, + "p90": 6.9036511005833745, + "p95": 6.945121941680554, + "p99": 7.021578256681096 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_4096_2048.jsonl" + }, + { + "name": "c64_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 97.68905732702115, + "request_throughput": 3.2756995384731473, + "input_token_throughput": 6916.199403360568, + "output_token_throughput": 3408.877197834196, + "total_token_throughput": 10325.076601194765, + "total_input_tokens": 675637, + "total_output_tokens": 333010, + "e2e_ms": { + "mean": 17265.869403236684, + "p50": 16820.79488999443, + "p90": 31243.420736011474, + "p95": 32918.48102155491, + "p99": 34336.54211862915 + }, + "ttft_ms": { + "mean": 162.13746994908433, + "p50": 91.81087499018759, + "p90": 534.6608572959667, + "p95": 570.7538236456458, + "p99": 582.0342552423244 + }, + "tpot_ms": { + "mean": 16.564744187293176, + "p50": 16.84016802253919, + "p90": 16.97582263718213, + "p95": 17.058498259752593, + "p99": 18.073979461173185 + }, + "itl_ms": { + "mean": 16.45873048013487, + "p50": 16.287181002553552, + "p90": 17.15008311439306, + "p95": 20.545937879069243, + "p99": 31.291485934634682 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_64_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 79.97539899300318, + "request_throughput": 0.06251922544878376, + "input_token_throughput": 124.66333554487483, + "output_token_throughput": 147.18276055152674, + "total_token_throughput": 271.84609609640154, + "total_input_tokens": 9970, + "total_output_tokens": 11771, + "e2e_ms": { + "mean": 15993.579583399696, + "p50": 14702.334433008218, + "p90": 23806.792291003512, + "p95": 24761.64468100178, + "p99": 25525.526593000395 + }, + "ttft_ms": { + "mean": 55.41398960631341, + "p50": 48.41957500320859, + "p90": 66.52183539699763, + "p95": 66.70366719481535, + "p99": 66.84913263306953 + }, + "tpot_ms": { + "mean": 6.770702241974898, + "p50": 6.757647574057017, + "p90": 6.794975151768566, + "p95": 6.7974891098195185, + "p99": 6.7995002762602805 + }, + "itl_ms": { + "mean": 6.772954078104776, + "p50": 6.770955995307304, + "p90": 6.9077145017217845, + "p95": 6.949156500922982, + "p99": 7.023812456463929 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_4096_4096.jsonl" + }, + { + "name": "c64_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 64, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 192.00843709899345, + "request_throughput": 1.6665934311783297, + "input_token_throughput": 3518.7880814407285, + "output_token_throughput": 3408.9439500126596, + "total_token_throughput": 6927.732031453388, + "total_input_tokens": 675637, + "total_output_tokens": 654546, + "e2e_ms": { + "mean": 33560.76050478714, + "p50": 33245.46831249609, + "p90": 59875.34798151173, + "p95": 65515.45453574946, + "p99": 68183.66740680416 + }, + "ttft_ms": { + "mean": 163.8100891470458, + "p50": 90.77135000552516, + "p90": 516.3448837934993, + "p95": 576.4098760992056, + "p99": 599.0004324031179 + }, + "tpot_ms": { + "mean": 16.438118558644927, + "p50": 16.6576844549002, + "p90": 16.753551188242543, + "p95": 16.788353501190862, + "p99": 16.975070930319287 + }, + "itl_ms": { + "mean": 16.346701673086326, + "p50": 16.38738051406108, + "p90": 16.858579698600806, + "p95": 17.46838169492548, + "p99": 29.494421854615243 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_64_4096_4096.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.7416468400042504, + "request_throughput": 1.3363099762762003, + "input_token_throughput": 11422.243153218451, + "output_token_throughput": 101.292296201736, + "total_token_throughput": 11523.535449420186, + "total_input_tokens": 42738, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 746.9622810021974, + "p50": 793.1687289965339, + "p90": 1112.0859866088722, + "p95": 1184.4593688088933, + "p99": 1242.3580745689105 + }, + "ttft_ms": { + "mean": 244.24475480336696, + "p50": 203.74058501329273, + "p90": 424.9969316122588, + "p95": 433.33762081456365, + "p99": 440.01017217640765 + }, + "tpot_ms": { + "mean": 6.720416888255487, + "p50": 6.736813561917737, + "p90": 6.792966854380781, + "p95": 6.798383077097527, + "p99": 6.802716055270923 + }, + "itl_ms": { + "mean": 6.738215297567921, + "p50": 6.794262997573242, + "p90": 6.934145401464775, + "p95": 6.970913801342249, + "p99": 7.334358422085639 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_16384_128.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 29.665646167995874, + "request_throughput": 5.3934439551366475, + "input_token_throughput": 43522.36228695046, + "output_token_throughput": 356.16955518733636, + "total_token_throughput": 43878.531842137796, + "total_input_tokens": 1291119, + "total_output_tokens": 10566, + "e2e_ms": { + "mean": 5789.677598388153, + "p50": 5906.899729496217, + "p90": 9852.575152096688, + "p95": 10661.965765064811, + "p99": 11830.348544723525 + }, + "ttft_ms": { + "mean": 996.1798031064973, + "p50": 711.6568210039986, + "p90": 2225.204480701359, + "p95": 3789.3565844627938, + "p99": 4532.54993600538 + }, + "tpot_ms": { + "mean": 78.85899350592113, + "p50": 76.84775757399144, + "p90": 116.30529798200597, + "p95": 150.73226668917212, + "p99": 226.61302751671002 + }, + "itl_ms": { + "mean": 74.18084010396575, + "p50": 13.344642997253686, + "p90": 252.1555712155532, + "p95": 263.4294065006543, + "p99": 274.03829396585934 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_32_16384_128.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.8248532990110107, + "request_throughput": 1.307239679308183, + "input_token_throughput": 11173.761882854626, + "output_token_throughput": 132.55410348184978, + "total_token_throughput": 11306.315986336476, + "total_input_tokens": 42738, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 763.5138192039449, + "p50": 759.5113300194498, + "p90": 1207.4323965993246, + "p95": 1299.0657877991907, + "p99": 1372.3725007590838 + }, + "ttft_ms": { + "mean": 87.44428500649519, + "p50": 80.63636301085353, + "p90": 121.7506884073373, + "p95": 122.3494662088342, + "p99": 122.82848845003173 + }, + "tpot_ms": { + "mean": 6.727216758879248, + "p50": 6.71551957022214, + "p90": 6.754395038357036, + "p95": 6.758074705418023, + "p99": 6.761018439066813 + }, + "itl_ms": { + "mean": 6.746741532935517, + "p50": 6.7992639960721135, + "p90": 6.96806499036029, + "p95": 7.014268980128691, + "p99": 7.566279004095122 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_16384_256.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 12.461962151981425, + "request_throughput": 12.839069646392751, + "input_token_throughput": 103604.79226738102, + "output_token_throughput": 1782.5443320310537, + "total_token_throughput": 105387.33659941207, + "total_input_tokens": 1291119, + "total_output_tokens": 22214, + "e2e_ms": { + "mean": 2229.4527406438647, + "p50": 2167.282502487069, + "p90": 3691.0632932878793, + "p95": 3832.958550745388, + "p99": 4262.988245337619 + }, + "ttft_ms": { + "mean": 200.0951993633862, + "p50": 141.25127698935103, + "p90": 556.7186772095738, + "p95": 636.4848169992911, + "p99": 644.9647748749703 + }, + "tpot_ms": { + "mean": 14.839397661739069, + "p50": 15.014273620808723, + "p90": 16.209308889668645, + "p95": 16.944263605099696, + "p99": 20.775335025510376 + }, + "itl_ms": { + "mean": 15.171862422589003, + "p50": 12.557223002659157, + "p90": 31.59359798883088, + "p95": 37.34407600131817, + "p99": 48.670664022210985 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_32_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 3.8126014840090647, + "request_throughput": 1.3114405009207388, + "input_token_throughput": 11209.668825670107, + "output_token_throughput": 132.9800667933629, + "total_token_throughput": 11342.64889246347, + "total_input_tokens": 42738, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 761.1607200058643, + "p50": 758.4194740047678, + "p90": 1202.8484933951404, + "p95": 1293.046706187306, + "p99": 1365.2052764210384 + }, + "ttft_ms": { + "mean": 85.19858720246702, + "p50": 78.2607370056212, + "p90": 117.92198420735076, + "p95": 118.64809161052108, + "p99": 119.22897753305733 + }, + "tpot_ms": { + "mean": 6.726653841894257, + "p50": 6.718013685991341, + "p90": 6.75291219102312, + "p95": 6.756540267141408, + "p99": 6.759442728036039 + }, + "itl_ms": { + "mean": 6.745665610796714, + "p50": 6.799466005759314, + "p90": 7.004150975262746, + "p95": 7.096040004398674, + "p99": 7.3483510059304535 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_16384_512.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 20.42796457701479, + "request_throughput": 7.832400501615778, + "input_token_throughput": 63203.50689528539, + "output_token_throughput": 1977.191601626633, + "total_token_throughput": 65180.698496912024, + "total_input_tokens": 1291119, + "total_output_tokens": 40390, + "e2e_ms": { + "mean": 3673.450332867833, + "p50": 3403.5250500019174, + "p90": 6646.781725299661, + "p95": 7038.214879437871, + "p99": 7504.8199216401545 + }, + "ttft_ms": { + "mean": 190.689647281215, + "p50": 130.69589500082657, + "p90": 422.3435439140301, + "p95": 640.862681595899, + "p99": 739.0641469051478 + }, + "tpot_ms": { + "mean": 14.041667680472107, + "p50": 14.111390889298962, + "p90": 14.77375568637589, + "p95": 15.699944199987575, + "p99": 18.530489535910686 + }, + "itl_ms": { + "mean": 14.017145062407957, + "p50": 12.583733012434095, + "p90": 24.489443507627584, + "p95": 33.16136909998022, + "p99": 42.72941159870243 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_32_16384_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 10.848079899995355, + "request_throughput": 0.4609110594771837, + "input_token_throughput": 3939.683371987175, + "output_token_throughput": 141.13096641191365, + "total_token_throughput": 4080.814338399089, + "total_input_tokens": 42738, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2168.263963004574, + "p50": 934.9753590067849, + "p90": 4636.084477195982, + "p95": 4767.81152759795, + "p99": 4873.193167919526 + }, + "ttft_ms": { + "mean": 96.11712900805287, + "p50": 109.78887401870452, + "p90": 129.60210220771842, + "p95": 132.0783076051157, + "p99": 134.05927192303352 + }, + "tpot_ms": { + "mean": 6.752114930746313, + "p50": 6.729682338092817, + "p90": 6.801452364504652, + "p95": 6.804286228679045, + "p99": 6.806553320018559 + }, + "itl_ms": { + "mean": 6.798231347786077, + "p50": 6.802242991398089, + "p90": 6.969152204692364, + "p95": 7.020317575370427, + "p99": 7.177459584490862 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 40.018767052009935, + "request_throughput": 3.9981241748916907, + "input_token_throughput": 32262.838041012405, + "output_token_throughput": 2122.3542416890677, + "total_token_throughput": 34385.19228270147, + "total_input_tokens": 1291119, + "total_output_tokens": 84934, + "e2e_ms": { + "mean": 7223.56558833817, + "p50": 7723.683465505019, + "p90": 12056.387461713166, + "p95": 13034.13519445021, + "p99": 13797.926251252065 + }, + "ttft_ms": { + "mean": 226.08372724425863, + "p50": 127.79736549418885, + "p90": 774.0484802838182, + "p95": 789.9365069199092, + "p99": 950.3970919386484 + }, + "tpot_ms": { + "mean": 13.375908518568897, + "p50": 13.288246175475457, + "p90": 13.820212318785108, + "p95": 14.232104893723827, + "p99": 22.015017151728067 + }, + "itl_ms": { + "mean": 13.27381102657931, + "p50": 12.61283500934951, + "p90": 13.946030507213436, + "p95": 22.966549753618892, + "p99": 39.84888230188517 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_32_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 38.62168169600773, + "request_throughput": 0.12946096028016418, + "input_token_throughput": 1106.5805040907312, + "output_token_throughput": 145.69536469929676, + "total_token_throughput": 1252.275868790028, + "total_input_tokens": 42738, + "total_output_tokens": 5627, + "e2e_ms": { + "mean": 7722.665159997996, + "p50": 7533.396830986021, + "p90": 11619.30468720384, + "p95": 11757.026566605782, + "p99": 11867.204070127336 + }, + "ttft_ms": { + "mean": 92.6008160051424, + "p50": 91.59555900259875, + "p90": 129.70803180942312, + "p95": 132.64805141370744, + "p99": 135.0000670971349 + }, + "tpot_ms": { + "mean": 6.767648317537002, + "p50": 6.796165545190339, + "p90": 6.811977873980548, + "p95": 6.816418280084744, + "p99": 6.819970604968102 + }, + "itl_ms": { + "mean": 6.787064156022911, + "p50": 6.7920129804406315, + "p90": 6.940115010365844, + "p95": 6.9823910016566515, + "p99": 7.056562218349427 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 82.30778569099493, + "request_throughput": 1.9439230281407651, + "input_token_throughput": 15686.474726062977, + "output_token_throughput": 2226.2535489403595, + "total_token_throughput": 17912.728275003337, + "total_input_tokens": 1291119, + "total_output_tokens": 183238, + "e2e_ms": { + "mean": 14894.456553130476, + "p50": 15637.624575494556, + "p90": 23955.132565906384, + "p95": 25291.544086861537, + "p99": 26578.48927184241 + }, + "ttft_ms": { + "mean": 187.66604684860795, + "p50": 125.363400002243, + "p90": 419.96127700840566, + "p95": 628.0171468533806, + "p99": 806.2383803006377 + }, + "tpot_ms": { + "mean": 12.880386136891058, + "p50": 13.052008063056935, + "p90": 13.237517172090145, + "p95": 13.394617533097497, + "p99": 14.850997615043472 + }, + "itl_ms": { + "mean": 12.886822602987632, + "p50": 12.698767008259892, + "p90": 13.1840615067631, + "p95": 14.173676259815693, + "p99": 34.75605665880721 + }, + "gpu_memory": { + "peak_used_mb": 132932.0, + "avg_used_mb": 132932.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_32_16384_2048.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.979828148992965, + "request_throughput": 0.8361444301442051, + "input_token_throughput": 29066.05268067286, + "output_token_throughput": 63.37974780493075, + "total_token_throughput": 29129.43242847779, + "total_input_tokens": 173810, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 1194.5084626029711, + "p50": 717.2065060003661, + "p90": 2381.096083810553, + "p95": 2550.764347409131, + "p99": 2686.4989582879934 + }, + "ttft_ms": { + "mean": 697.6236455957405, + "p50": 247.50089697772637, + "p90": 1633.7114335969093, + "p95": 1783.813636796549, + "p99": 1903.895399356261 + }, + "tpot_ms": { + "mean": 6.668602271081101, + "p50": 6.7253280998556875, + "p90": 6.751154703156585, + "p95": 6.75149141538628, + "p99": 6.7517607851700365 + }, + "itl_ms": { + "mean": 6.824716656570512, + "p50": 6.875582999782637, + "p90": 7.054750301176682, + "p95": 7.115953753236681, + "p99": 7.555937078141144 + }, + "gpu_memory": { + "peak_used_mb": 133260.0, + "avg_used_mb": 133009.68421052632, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_65536_128.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 39.645488456008025, + "request_throughput": 1.0089420400100595, + "input_token_throughput": 37118.27139254208, + "output_token_throughput": 64.77407896864582, + "total_token_throughput": 37183.045471510726, + "total_input_tokens": 1471572, + "total_output_tokens": 2568, + "e2e_ms": { + "mean": 7821.816953149391, + "p50": 7532.596676493995, + "p90": 14040.57804098993, + "p95": 14289.917052967932, + "p99": 16484.16690281534 + }, + "ttft_ms": { + "mean": 3209.808414100553, + "p50": 2829.1647829901194, + "p90": 5736.555838398636, + "p95": 7445.337255309275, + "p99": 8645.530720362147 + }, + "tpot_ms": { + "mean": 88.42192167431719, + "p50": 79.14389640022598, + "p90": 163.34168494834844, + "p95": 248.75438660297624, + "p99": 263.8012609491124 + }, + "itl_ms": { + "mean": 83.73990312849202, + "p50": 8.835267013637349, + "p90": 285.96494658268057, + "p95": 294.62111500033643, + "p99": 342.85850628395565 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133307.27272727274, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_8_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.205351618991699, + "request_throughput": 1.1889612220342305, + "input_token_throughput": 41330.670000353915, + "output_token_throughput": 120.56066791427097, + "total_token_throughput": 41451.23066826819, + "total_input_tokens": 173810, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 839.7061410069, + "p50": 868.7234710087068, + "p90": 1248.29141360824, + "p95": 1306.3532338070215, + "p99": 1352.8026899660472 + }, + "ttft_ms": { + "mean": 161.62108960561454, + "p50": 179.29533100686967, + "p90": 252.30673381011005, + "p95": 254.46968341129832, + "p99": 256.20004309224896 + }, + "tpot_ms": { + "mean": 6.735750522493605, + "p50": 6.755977851234777, + "p90": 6.784939020019157, + "p95": 6.793552222764636, + "p99": 6.80044278496102 + }, + "itl_ms": { + "mean": 6.807582022121501, + "p50": 6.856819003587589, + "p90": 7.1003060875227675, + "p95": 7.191569736460224, + "p99": 7.500103442289398 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_65536_256.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 8.643470701994374, + "request_throughput": 4.627770646665175, + "input_token_throughput": 170252.44265135913, + "output_token_throughput": 608.089062971804, + "total_token_throughput": 170860.53171433092, + "total_input_tokens": 1471572, + "total_output_tokens": 5256, + "e2e_ms": { + "mean": 1581.5019286506868, + "p50": 1422.0479399955366, + "p90": 2723.607920407085, + "p95": 2865.085011156043, + "p99": 2998.3723240473773 + }, + "ttft_ms": { + "mean": 356.03623832575977, + "p50": 322.12173151492607, + "p90": 653.8560661108933, + "p95": 749.1073189507006, + "p99": 864.765941716323 + }, + "tpot_ms": { + "mean": 9.142763488004903, + "p50": 9.527620677225901, + "p90": 10.037834474508111, + "p95": 10.212376501737555, + "p99": 10.296325263987136 + }, + "itl_ms": { + "mean": 11.325642503234223, + "p50": 8.695702519617043, + "p90": 15.801905601983892, + "p95": 25.98413567902753, + "p99": 93.4788243641378 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_8_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.22689065898885, + "request_throughput": 1.182902611726439, + "input_token_throughput": 41120.06058883447, + "output_token_throughput": 119.94632482906093, + "total_token_throughput": 41240.006913663536, + "total_input_tokens": 173810, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 843.982104404131, + "p50": 873.4139300067909, + "p90": 1252.0642684074119, + "p95": 1310.3560922085308, + "p99": 1356.9895512494259 + }, + "ttft_ms": { + "mean": 184.7932830103673, + "p50": 207.99873900250532, + "p90": 287.43030800833367, + "p95": 288.96910000476055, + "p99": 290.20013360190205 + }, + "tpot_ms": { + "mean": 6.539759853660324, + "p50": 6.52367834317927, + "p90": 6.738242484397563, + "p95": 6.744176779429061, + "p99": 6.748924215454261 + }, + "itl_ms": { + "mean": 6.809284541349904, + "p50": 6.858499997179024, + "p90": 7.103115899371915, + "p95": 7.176107993291225, + "p99": 7.463799362885767 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_65536_512.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 15.791608511004597, + "request_throughput": 2.5329908585389167, + "input_token_throughput": 93186.96059204577, + "output_token_throughput": 705.6912531889423, + "total_token_throughput": 93892.65184523471, + "total_input_tokens": 1471572, + "total_output_tokens": 11144, + "e2e_ms": { + "mean": 2813.447674074996, + "p50": 3025.5881055054488, + "p90": 4910.965449296054, + "p95": 5054.251725919312, + "p99": 5200.705566430115 + }, + "ttft_ms": { + "mean": 274.49545034905896, + "p50": 277.5276810134528, + "p90": 459.80448708287463, + "p95": 661.3905629361398, + "p99": 679.1221809445415 + }, + "tpot_ms": { + "mean": 9.104545633011229, + "p50": 9.216829093139399, + "p90": 9.54288235983806, + "p95": 9.628486301198969, + "p99": 10.393052560959216 + }, + "itl_ms": { + "mean": 10.05413049895824, + "p50": 8.68834598804824, + "p90": 10.545664001256227, + "p95": 16.518691991223022, + "p99": 69.51865501469001 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_8_65536_512.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 11.221458473999519, + "request_throughput": 0.44557487884352653, + "input_token_throughput": 15489.07393835867, + "output_token_throughput": 136.43502790188782, + "total_token_throughput": 15625.508966260559, + "total_input_tokens": 173810, + "total_output_tokens": 1531, + "e2e_ms": { + "mean": 2242.899997602217, + "p50": 1069.672420999268, + "p90": 4670.665889407974, + "p95": 4765.812398202252, + "p99": 4841.929605237674 + }, + "ttft_ms": { + "mean": 161.17175800609402, + "p50": 174.0668680286035, + "p90": 253.8997896015644, + "p95": 254.4195618014783, + "p99": 254.83537956140933 + }, + "tpot_ms": { + "mean": 6.757404848240741, + "p50": 6.754809487582192, + "p90": 6.844067327086863, + "p95": 6.851316639924977, + "p99": 6.857116090195467 + }, + "itl_ms": { + "mean": 6.8386421748008654, + "p50": 6.861203015432693, + "p90": 7.040805980795994, + "p95": 7.1101105568232015, + "p99": 7.256177014496642 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 29.799556770012714, + "request_throughput": 1.3423018439070205, + "input_token_throughput": 49382.34522604855, + "output_token_throughput": 734.776029354703, + "total_token_throughput": 50117.121255403246, + "total_input_tokens": 1471572, + "total_output_tokens": 21896, + "e2e_ms": { + "mean": 5146.086770098918, + "p50": 5226.150454007438, + "p90": 8645.80303388648, + "p95": 8994.806143906315, + "p99": 9476.986618578085 + }, + "ttft_ms": { + "mean": 281.65961030026665, + "p50": 266.0276324895676, + "p90": 418.90704880934237, + "p95": 671.0752429964487, + "p99": 707.7661068245652 + }, + "tpot_ms": { + "mean": 8.906683843567604, + "p50": 8.975047370547262, + "p90": 9.154302167481521, + "p95": 9.196039805515325, + "p99": 10.493496254532058 + }, + "itl_ms": { + "mean": 9.247891059879954, + "p50": 8.68852600979153, + "p90": 8.970274080638774, + "p95": 11.745035581407134, + "p99": 36.71318077569595 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_8_65536_1024.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 9.016311184008373, + "request_throughput": 0.5545505138363198, + "input_token_throughput": 41083.09844684438, + "output_token_throughput": 42.03492894879304, + "total_token_throughput": 41125.13337579317, + "total_input_tokens": 370418, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 1801.840906200232, + "p50": 962.3327779991087, + "p90": 3719.553761993302, + "p95": 3782.0753349922597, + "p99": 3832.092593391426 + }, + "ttft_ms": { + "mean": 1332.752106199041, + "p50": 549.2673270055093, + "p90": 3005.3896251891274, + "p95": 3042.4025475862436, + "p99": 3072.0128855039366 + }, + "tpot_ms": { + "mean": 6.340339875959231, + "p50": 6.323813537255022, + "p90": 6.668300853925757, + "p95": 6.696194727555849, + "p99": 6.718509826459921 + }, + "itl_ms": { + "mean": 6.877446249284429, + "p50": 6.9577679969370365, + "p90": 7.167970004957169, + "p95": 7.233284006360918, + "p99": 8.599375199992219 + }, + "gpu_memory": { + "peak_used_mb": 133780.0, + "avg_used_mb": 133780.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_131072_128.jsonl" + }, + { + "name": "c4_i131072_o128", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 24.360167828010162, + "request_throughput": 0.8210124060394736, + "input_token_throughput": 60525.075623850295, + "output_token_throughput": 46.75665652394802, + "total_token_throughput": 60571.83228037424, + "total_input_tokens": 1474401, + "total_output_tokens": 1139, + "e2e_ms": { + "mean": 4786.172839647043, + "p50": 5626.231432994246, + "p90": 8825.416628399398, + "p95": 9054.740018452867, + "p99": 9263.157454879838 + }, + "ttft_ms": { + "mean": 2268.90818515094, + "p50": 1815.796633993159, + "p90": 5463.91780730628, + "p95": 5665.4495449300175, + "p99": 7563.445302569886 + }, + "tpot_ms": { + "mean": 46.91910022441427, + "p50": 8.155252634180076, + "p90": 111.22875968014293, + "p95": 130.55081088415008, + "p99": 135.39232037676058 + }, + "itl_ms": { + "mean": 60.29291451981146, + "p50": 7.773411023663357, + "p90": 318.7832578085363, + "p95": 342.509480001172, + "p99": 410.38581036787946 + }, + "gpu_memory": { + "peak_used_mb": 134302.0, + "avg_used_mb": 134005.17647058822, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_4_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.8524470679985825, + "request_throughput": 1.030407942618172, + "input_token_throughput": 76336.32985774761, + "output_token_throughput": 104.48336538148263, + "total_token_throughput": 76440.81322312908, + "total_input_tokens": 370418, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 969.0659300016705, + "p50": 1085.9827550011687, + "p90": 1327.6738234155346, + "p95": 1348.4903072181623, + "p99": 1365.1434942602646 + }, + "ttft_ms": { + "mean": 324.2844178050291, + "p50": 446.85381901217625, + "p90": 519.0396234043874, + "p95": 526.125369197689, + "p99": 531.7939658323303 + }, + "tpot_ms": { + "mean": 6.38195852240975, + "p50": 6.3424957519270135, + "p90": 6.734525240305068, + "p95": 6.740891553716675, + "p99": 6.745984604445959 + }, + "itl_ms": { + "mean": 6.873506368875111, + "p50": 6.902034016093239, + "p90": 7.1110547869466245, + "p95": 7.182737207040191, + "p99": 8.356347646331415 + }, + "gpu_memory": { + "peak_used_mb": 134302.0, + "avg_used_mb": 134302.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_131072_256.jsonl" + }, + { + "name": "c4_i131072_o256", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 8.323286791011924, + "request_throughput": 2.402896896643934, + "input_token_throughput": 177141.67936543564, + "output_token_throughput": 306.008919787605, + "total_token_throughput": 177447.68828522324, + "total_input_tokens": 1474401, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 1532.3867275510565, + "p50": 1609.7988390101818, + "p90": 2462.9763679899047, + "p95": 2573.004158611002, + "p99": 2602.991411728144 + }, + "ttft_ms": { + "mean": 542.0563667998067, + "p50": 561.1400620109634, + "p90": 786.9597911747405, + "p95": 826.1095973968623, + "p99": 873.837375488656 + }, + "tpot_ms": { + "mean": 7.110337367266883, + "p50": 7.7316950582082455, + "p90": 8.280846323565472, + "p95": 8.423364776975943, + "p99": 8.690418247445379 + }, + "itl_ms": { + "mean": 10.038461456174083, + "p50": 7.76114099426195, + "p90": 13.23855118243955, + "p95": 20.75249279732816, + "p99": 85.82791325403365 + }, + "gpu_memory": { + "peak_used_mb": 134302.0, + "avg_used_mb": 134302.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_4_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 4.870791513007134, + "request_throughput": 1.0265272054137038, + "input_token_throughput": 76048.83087498667, + "output_token_throughput": 104.08985862894957, + "total_token_throughput": 76152.92073361563, + "total_input_tokens": 370418, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 972.7444024058059, + "p50": 1082.7535319840536, + "p90": 1337.6641850161832, + "p95": 1353.217571019195, + "p99": 1365.6602798216045 + }, + "ttft_ms": { + "mean": 327.9147273977287, + "p50": 443.6211699794512, + "p90": 527.4083624070045, + "p95": 528.5839962074533, + "p99": 529.5245032478124 + }, + "tpot_ms": { + "mean": 6.384223606274097, + "p50": 6.339855917367684, + "p90": 6.738858764336018, + "p95": 6.743368868929076, + "p99": 6.746976952603522 + }, + "itl_ms": { + "mean": 6.8740309701973334, + "p50": 6.8926950043533, + "p90": 7.111179392086342, + "p95": 7.18118479126133, + "p99": 8.424099116818978 + }, + "gpu_memory": { + "peak_used_mb": 134302.0, + "avg_used_mb": 134302.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_131072_512.jsonl" + }, + { + "name": "c4_i131072_o512", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 15.363296866009478, + "request_throughput": 1.3018039145132316, + "input_token_throughput": 95969.04966811115, + "output_token_throughput": 382.40489988826175, + "total_token_throughput": 96351.45456799942, + "total_input_tokens": 1474401, + "total_output_tokens": 5875, + "e2e_ms": { + "mean": 2674.266563852143, + "p50": 2755.7085349981207, + "p90": 3774.087236606283, + "p95": 3889.944781288796, + "p99": 4365.159926651103 + }, + "ttft_ms": { + "mean": 384.13389480265323, + "p50": 420.3916975093307, + "p90": 557.5033120956505, + "p95": 581.2123146577505, + "p99": 686.7281405249375 + }, + "tpot_ms": { + "mean": 7.672143939324046, + "p50": 7.918229260161499, + "p90": 8.181102878817748, + "p95": 8.271288068884912, + "p99": 8.300791830289597 + }, + "itl_ms": { + "mean": 8.881502143883546, + "p50": 7.747493014903739, + "p90": 8.15649639698677, + "p95": 13.578455609967932, + "p99": 36.924229603027925 + }, + "gpu_memory": { + "peak_used_mb": 134302.0, + "avg_used_mb": 134302.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_4_131072_512.jsonl" + }, + { + "name": "c1_i262144_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 21.95894926597248, + "request_throughput": 0.22769759788770882, + "input_token_throughput": 34775.52549307653, + "output_token_throughput": 17.25947791988833, + "total_token_throughput": 34792.784970996414, + "total_input_tokens": 763634, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 4389.704544801498, + "p50": 5417.287694988772, + "p90": 7652.186370198615, + "p95": 8216.366721596569, + "p99": 8667.711002714932 + }, + "ttft_ms": { + "mean": 3949.9581184005365, + "p50": 5366.542825999204, + "p90": 7064.6827823948115, + "p95": 7547.155738197034, + "p99": 7933.134102838812 + }, + "tpot_ms": { + "mean": 5.314242966814787, + "p50": 5.8620529295220765, + "p90": 6.238149659580012, + "p95": 6.248887207276749, + "p99": 6.257477245434139 + }, + "itl_ms": { + "mean": 7.023914325863314, + "p50": 7.034406007733196, + "p90": 7.234651799080893, + "p95": 7.318750605918467, + "p99": 11.975427612196649 + }, + "gpu_memory": { + "peak_used_mb": 134304.0, + "avg_used_mb": 134302.73913043478, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_262144_128.jsonl" + }, + { + "name": "c2_i262144_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 17.299146233010106, + "request_throughput": 0.5780632099009645, + "input_token_throughput": 80576.28863441643, + "output_token_throughput": 43.701578668512916, + "total_token_throughput": 80619.99021308494, + "total_input_tokens": 1393901, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 2911.29859109933, + "p50": 1588.1151799985673, + "p90": 7329.382639605319, + "p95": 7493.520934291882, + "p99": 7624.831570041133 + }, + "ttft_ms": { + "mean": 1811.4683610998327, + "p50": 704.2788730032044, + "p90": 5830.754057096783, + "p95": 6405.292088544227, + "p99": 6864.922513702186 + }, + "tpot_ms": { + "mean": 12.037992758067379, + "p50": 6.531740300510898, + "p90": 13.113503143811732, + "p95": 40.837272541715954, + "p99": 63.01628806003945 + }, + "itl_ms": { + "mean": 18.799718608482717, + "p50": 7.143330993130803, + "p90": 12.45063220849261, + "p95": 20.016222586855275, + "p99": 397.8567520412612 + }, + "gpu_memory": { + "peak_used_mb": 134304.0, + "avg_used_mb": 134304.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_2_262144_128.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 5.492067698010942, + "request_throughput": 0.9104039270693707, + "input_token_throughput": 139043.07848873836, + "output_token_throughput": 92.31495820483418, + "total_token_throughput": 139135.3934469432, + "total_input_tokens": 763634, + "total_output_tokens": 507, + "e2e_ms": { + "mean": 1096.547552390257, + "p50": 977.0184349908959, + "p90": 1619.9407393927686, + "p95": 1644.6769221976865, + "p99": 1664.4658684416208 + }, + "ttft_ms": { + "mean": 483.5241790045984, + "p50": 427.5413209979888, + "p90": 664.6515952073969, + "p95": 741.0604366043117, + "p99": 802.1875097218435 + }, + "tpot_ms": { + "mean": 5.448600248984309, + "p50": 6.018701760138747, + "p90": 6.444691271133902, + "p95": 6.504228704678525, + "p99": 6.551858651514223 + }, + "itl_ms": { + "mean": 6.981518899808173, + "p50": 7.025405997410417, + "p90": 7.230082410387695, + "p95": 7.303182306350208, + "p99": 11.031649834476413 + }, + "gpu_memory": { + "peak_used_mb": 134304.0, + "avg_used_mb": 134304.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_262144_256.jsonl" + }, + { + "name": "c2_i262144_o256", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 8.155221176013583, + "request_throughput": 1.226208312953221, + "input_token_throughput": 170921.2993633808, + "output_token_throughput": 171.17868048826966, + "total_token_throughput": 171092.47804386905, + "total_input_tokens": 1393901, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 1528.3535980037414, + "p50": 1439.8152739886427, + "p90": 1988.2277190132297, + "p95": 2101.097439012665, + "p99": 2191.3932150122127 + }, + "ttft_ms": { + "mean": 588.1965045962716, + "p50": 540.3380134957843, + "p90": 763.7523676938143, + "p95": 872.867106342164, + "p99": 960.1588972608442 + }, + "tpot_ms": { + "mean": 6.689213261609457, + "p50": 6.6852245617160255, + "p90": 7.057503772096951, + "p95": 7.161944660791931, + "p99": 7.2454973717479145 + }, + "itl_ms": { + "mean": 8.125442223882589, + "p50": 7.247152010677382, + "p90": 7.577662786934525, + "p95": 11.998600204242393, + "p99": 27.029610665049447 + }, + "gpu_memory": { + "peak_used_mb": 134304.0, + "avg_used_mb": 134304.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_2_262144_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 21.47346921497956, + "request_throughput": 0.23284546851479768, + "input_token_throughput": 47769.55180043443, + "output_token_throughput": 17.649686513421663, + "total_token_throughput": 47787.201486947844, + "total_input_tokens": 1025778, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 4292.933715006802, + "p50": 812.884353013942, + "p90": 11330.1644160063, + "p95": 14596.886557008835, + "p99": 17210.26426981087 + }, + "ttft_ms": { + "mean": 3868.1056150060613, + "p50": 447.73642800282687, + "p90": 10681.944887607826, + "p95": 13982.401888811726, + "p99": 16622.76748977485 + }, + "tpot_ms": { + "mean": 5.175154858090192, + "p50": 5.693625568585726, + "p90": 5.999625637622856, + "p95": 6.096606657665045, + "p99": 6.174191473698796 + }, + "itl_ms": { + "mean": 7.127220278598787, + "p50": 7.100967501173727, + "p90": 7.371948787476867, + "p95": 7.5303360528778285, + "p99": 13.657342679798576 + }, + "gpu_memory": { + "peak_used_mb": 134304.0, + "avg_used_mb": 134304.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_524288_128.jsonl" + }, + { + "name": "c2_i524288_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 5.669274649990257, + "request_throughput": 1.7638940812326294, + "input_token_throughput": 292108.79737548897, + "output_token_throughput": 133.35039254118678, + "total_token_throughput": 292242.1477680301, + "total_input_tokens": 1656045, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 1126.3942198012955, + "p50": 1179.683152004145, + "p90": 1650.5098093184642, + "p95": 1682.9271176538896, + "p99": 1708.8609643222298 + }, + "ttft_ms": { + "mean": 645.4665984987514, + "p50": 532.4026479938766, + "p90": 1085.0829599949066, + "p95": 1167.1218074916394, + "p99": 1232.752885489026 + }, + "tpot_ms": { + "mean": 5.6435321089686115, + "p50": 6.41863733328779, + "p90": 6.854364659400855, + "p95": 6.899553045213209, + "p99": 6.935703753863092 + }, + "itl_ms": { + "mean": 9.410653624309084, + "p50": 7.158250984502956, + "p90": 12.58740501361899, + "p95": 18.499862009775825, + "p99": 67.42094629444139 + }, + "gpu_memory": { + "peak_used_mb": 134306.0, + "avg_used_mb": 134304.06666666668, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_2_524288_128.jsonl" + }, + { + "name": "c1_i1048576_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1048576, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 52.218247757002246, + "request_throughput": 0.09575196822512148, + "input_token_throughput": 29684.37407576823, + "output_token_throughput": 7.257999191464208, + "total_token_throughput": 29691.632074959696, + "total_input_tokens": 1550066, + "total_output_tokens": 379, + "e2e_ms": { + "mean": 10442.118058208143, + "p50": 1504.5330520079006, + "p90": 29315.70396400057, + "p95": 38536.12910800147, + "p99": 45912.4692232022 + }, + "ttft_ms": { + "mean": 10048.045438004192, + "p50": 768.1465140194632, + "p90": 28969.904984202007, + "p95": 38265.961359604255, + "p99": 45702.80645992607 + }, + "tpot_ms": { + "mean": 4.670817527462662, + "p50": 5.614634058849119, + "p90": 5.992841449196221, + "p95": 6.039340303062727, + "p99": 6.076539386155933 + }, + "itl_ms": { + "mean": 7.190266562061534, + "p50": 7.141470996430144, + "p90": 7.409438196918927, + "p95": 7.556129548174793, + "p99": 12.29528843308791 + }, + "gpu_memory": { + "peak_used_mb": 134310.0, + "avg_used_mb": 134307.47368421053, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_1_1048576_128.jsonl" + }, + { + "name": "c2_i1048576_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 1048576, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 122.7578322670015, + "request_throughput": 0.08146119734543486, + "input_token_throughput": 26303.07932594507, + "output_token_throughput": 6.158466519314875, + "total_token_throughput": 26309.237792464384, + "total_input_tokens": 3228909, + "total_output_tokens": 756, + "e2e_ms": { + "mean": 24549.48970649275, + "p50": 4756.214266992174, + "p90": 61223.59725128044, + "p95": 63918.14225814305, + "p99": 66073.77826363314 + }, + "ttft_ms": { + "mean": 23067.16758919356, + "p50": 4525.840376983979, + "p90": 59377.30671290483, + "p95": 59936.56773844705, + "p99": 60383.97655888082 + }, + "tpot_ms": { + "mean": 15.730498989072503, + "p50": 5.885290517918038, + "p90": 41.93317375993312, + "p95": 56.69309552073888, + "p99": 68.50103292938354 + }, + "itl_ms": { + "mean": 26.2815643422692, + "p50": 7.6735740003641695, + "p90": 15.467662602895867, + "p95": 269.72740284109045, + "p99": 347.4504445269122 + }, + "gpu_memory": { + "peak_used_mb": 134312.0, + "avg_used_mb": 134310.22972972973, + "peak_utilization_pct": 100.0, + "memory_total_mb": 143771.0 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/raw_outputs/vllm_main_2_1048576_128.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/scenarios.tsv b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/scenarios.tsv new file mode 100644 index 0000000..3717ac1 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/results/20260709-152419_matrix/tp8_dp1/scenarios.tsv @@ -0,0 +1,57 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 +Y 1048576 128 1 5 +Y 1048576 128 2 10 From 938b2f1e88f812cb085cadb4e7c5ca29e237df98 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:22:30 +0000 Subject: [PATCH 29/53] workflow: document what to commit for each experiment Add rule 10: commit experiment code together with final results.json and .md reports; do not commit logs/raw_outputs/gpu_logs. Also update the pre-commit checklist accordingly. --- BENCHMARK_WORKFLOW.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/BENCHMARK_WORKFLOW.md b/BENCHMARK_WORKFLOW.md index e36f593..ee298cc 100644 --- a/BENCHMARK_WORKFLOW.md +++ b/BENCHMARK_WORKFLOW.md @@ -83,6 +83,12 @@ - This is required for cross-platform reproduction: when the same experiment is run on H200 and P800, the only differences should be model paths, ports, and device IDs. - If an experiment uses `start_server.sh`, that script should be self-contained and its command line should be reproducible from `results.json` alone. +10. **Version control: commit code and final artifacts only.** + - Always commit the experiment code (`config.env`, `run_bench.sh`, `start_*.sh`, parsers, etc.) together with the run's final artifacts. + - Final artifacts to commit: `results.json` (structured data) and `report.md` / `comparison.md` (human-readable summaries). + - Do **not** commit intermediate logs, per-request JSONL raw outputs, or GPU sampling CSVs. These are already ignored by `.gitignore` (`raw_outputs/`, `logs/`, `gpu_logs/`). + - `scenarios.tsv` and `skipped_after_oom.csv` may be committed if they are useful for reproducing the test plan, but they are optional. + ## Naming Conventions ### Experiment result directories @@ -340,3 +346,4 @@ See `docs/H200_QUICKSTART.md` for a concrete H200 porting walkthrough. - [ ] `/README.md` exists and documents provenance (or the report itself covers provenance). - [ ] Scripts either live under `experiments//` or in `scripts/` (or `scripts//`). - [ ] Script path references updated after moving. +- [ ] Commits include the experiment code plus final `results.json` and `.md` reports; logs/raw outputs are left ignored. From 4ea6e92b4232a259ab9c997d16936c2a39003ca5 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:51:29 +0000 Subject: [PATCH 30/53] sglang tp_dp_matrix: fix timeout calling bash function (rc=127) timeout(1) cannot execute shell functions directly; it needs a standalone command. Export run_bench_serving via export -f and wrap both warmup and scenario runs in bash -c so the function is visible in the subshell. --- .../run_bench.sh | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 09a5e09..2291b8d 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -132,26 +132,29 @@ run_bench_serving() { "$PYTHON" -m sglang.bench_serving "$@" fi } +export -f run_bench_serving run_warmup() { local input_len="$1" local output_len="$2" log "warming up (input=${input_len}, output=${output_len}, num=1)" - run_bench_serving \ - --backend sglang \ - --host 127.0.0.1 \ - --port "$SGLANG_PORT" \ - --dataset-name random \ - --dataset-path "$DATASET_PATH" \ - --random-input-len "$input_len" \ - --random-output-len "$output_len" \ - --num-prompts 1 \ - --max-concurrency 1 \ - --request-rate 10000 \ - --output-file /dev/null \ - --output-details \ - >> "${log_dir_global}/warmup.log" 2>&1 + bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' log "warmup completed" } @@ -415,21 +418,22 @@ run_parallel_config() { gpu_pid="$(start_gpu_monitor "$gpu_csv")" bench_rc=0 - timeout "$SCENARIO_TIMEOUT_S" \ + timeout "$SCENARIO_TIMEOUT_S" bash -c ' run_bench_serving \ --backend sglang \ --host 127.0.0.1 \ - --port "$SGLANG_PORT" \ + --port "'"$SGLANG_PORT"'" \ --dataset-name random \ - --dataset-path "$DATASET_PATH" \ - --random-input-len "$isl" \ - --random-output-len "$dsl" \ - --num-prompts "$num" \ - --max-concurrency "$conc" \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$dsl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ --request-rate 10000 \ - --output-file "$output_file" \ + --output-file "'"$output_file"'" \ --output-details \ - > "$detail_log" 2>&1 || bench_rc=$? + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? stop_gpu_monitor "$gpu_pid" From 605b976fb82e0bda1144c2e7bed503491be059a1 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 03:13:11 +0000 Subject: [PATCH 31/53] sglang tp_dp_matrix: fix empty DATASET_PATH causing docker invalid spec When DATASET_PATH is empty, -v ::ro is passed to docker run, which produces 'invalid spec: ::ro: empty section between colons'. Build the volume args dynamically and only add the dataset mount when the path is non-empty. --- experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 2291b8d..3f3bb3f 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -120,11 +120,15 @@ restart_server() { run_bench_serving() { # Run sglang.bench_serving either natively or inside the SGLang Docker image. if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") docker run --rm \ --network host \ - -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ - -v "${DATASET_PATH}:${DATASET_PATH}:ro" \ - -v "${RESULT_BASE}:${RESULT_BASE}" \ + "${vol_args[@]}" \ -e PYTHONUNBUFFERED=1 \ "${DOCKER_IMAGE}" \ python -m sglang.bench_serving "$@" From 58a3616872bc27ac4c45f82c59d8c0b8892bfc8f Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 04:07:14 +0000 Subject: [PATCH 32/53] sglang tp_dp_matrix: export DATASET_PATH for bash -c subshells export -f only exports the function definition, not the variables it references. DATASET_PATH was empty inside timeout bash -c subshells, causing docker run -v ::ro invalid spec even after the dynamic vol_args fix. Add explicit export after sourcing config.env. --- experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 3f3bb3f..619d25d 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -12,6 +12,9 @@ source "${SCRIPT_DIR}/../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH + RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" From e62ab7c970673f556a844c8e57a4ae20771ab1dd Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 04:57:22 +0000 Subject: [PATCH 33/53] sglang tp_dp_matrix: export MODEL_PATH and RESULT_BASE for bash -c subshells export -f only exports the function definition, not the variables it references. MODEL_PATH and RESULT_BASE were empty inside timeout bash -c subshells, causing docker run -v ::ro invalid spec. Add explicit export for all variables used by run_bench_serving(). --- experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 619d25d..a1bc182 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -13,7 +13,7 @@ source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/config.env" # Export variables used inside functions that are called via bash -c subshells. -export DATASET_PATH +export DATASET_PATH MODEL_PATH RESULT_BASE RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" From db00c1b1ac93f944ec00b30094e994540fc16b27 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Fri, 10 Jul 2026 09:06:10 +0000 Subject: [PATCH 34/53] Add custom benchmark setup for vLLM TP=2 on dsv4_h200 - Create configuration file for dsv4_h200_vllm_tp2_custom_bench. - Implement result parsing script to handle JSONL outputs from bench_client.py. - Develop run_bench.sh script to orchestrate multi-service vLLM benchmarking. - Add start_server.sh script to launch independent vLLM services on specified GPU pairs. --- .../dsv4_h200_vllm_tp2_custom_bench/README.md | 271 ++++++++ .../bench_client.py | 617 ++++++++++++++++++ .../config.env | 75 +++ .../parse_results.py | 206 ++++++ .../run_bench.sh | 277 ++++++++ .../start_server.sh | 109 ++++ 6 files changed, 1555 insertions(+) create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/README.md create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/config.env create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/README.md b/experiments/dsv4_h200_vllm_tp2_custom_bench/README.md new file mode 100644 index 0000000..20421ee --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/README.md @@ -0,0 +1,271 @@ +# dsv4_h200_vllm_tp2_custom_bench + +## 目的 + +当 vLLM 使用 **TP=2**（Tensor Parallel = 2）部署时，单张 H200 上可以同时运行 **4 个独立服务**（8 卡 / 2 = 4 服务），每个服务独占 2 张 GPU： + +| 服务 | 端口 | GPU 对 | +|------|------|--------| +| 1 | 30005 | 0, 1 | +| 2 | 30006 | 2, 3 | +| 3 | 30007 | 4, 5 | +| 4 | 30008 | 6, 7 | + +此时 `sglang.bench_serving` 存在以下问题： + +1. **低并发下无法测出真实吞吐**：并发数不大时，请求可能只落在一个服务上，其他 3 个服务空闲，bench 报告的吞吐只是单服务的 1/4。 +2. **TTFT 测量不够精确**：`sglang.bench_serving` 的计时粒度较粗，对于低延迟场景（TP=2 下 TTFT 可能 < 100ms）误差较大。 +3. **不支持多服务负载均衡**：只能测一个端口，无法反映整个 8 卡集群性能。 + +因此本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API，通过 `aiohttp` 并发发送请求，并记录每个请求的精确时间戳。同时通过 **负载均衡** 将请求均匀分发到 4 个服务上。 + +## 与 sglang.bench_serving 的区别 + +| 特性 | sglang.bench_serving | bench_client.py (本实验) | +|------|----------------------|--------------------------| +| 协议 | 内部协议 | OpenAI API (`/v1/chat/completions`) | +| 并发控制 | 进程级 | `asyncio.Semaphore` | +| TTFT 测量 | 粗略 | 精确到 `time.perf_counter()` | +| TPOT 测量 | 基于总时间估算 | 基于首 token 后每个 token 的间隔 | +| 多服务支持 | ❌ 单端口 | ✅ 轮询/随机分发到多个端口 | +| 低并发精度 | 一般 | 高（适合 TP=2 场景） | +| 集群吞吐 | 只能测单服务 | 可测 4 服务总吞吐 | + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡 | +| `config.env` | 实验配置（场景、模型路径、端口列表、LB 策略等） | +| `run_bench.sh` | 编排脚本：启动 4 个服务 → 运行压测 → 解析结果 | +| `start_server.sh` | 同时启动 4 个 vLLM TP=2 服务（每服务 2 GPU） | +| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | + +## 快速运行 + +```bash +# 完整流程（自动启动 4 个服务、压测、生成报告） +bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh + +# 跳过服务管理（已有 4 个服务在运行） +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh + +# 单独运行压测客户端（多服务，带健康检查） +python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ + --lb-strategy round_robin \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file /tmp/test.jsonl + +# 单独运行压测客户端（单服务，向后兼容） +python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ + --output-file /tmp/test_single.jsonl + +# 解析已有结果 +python3 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py \ + experiments/dsv4_h200_vllm_tp2_custom_bench/results/ +``` + +## 场景设计 + +本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： + +| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | +|------|----------|----------|--------|------| +| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | +| 4 | 512 | 256 | 80 | 低并发，4 服务各 1 req | +| 8 | 512 | 256 | 160 | 中低并发 | +| 16 | 512 | 256 | 320 | 中等并发 | +| 32 | 512 | 256 | 640 | 中高并发 | +| 64 | 512 | 256 | 1280 | 高并发 | +| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | +| 1 | 2048 | 512 | 20 | 长输入基线 | +| 8 | 2048 | 512 | 160 | 长输入中并发 | +| 32 | 2048 | 512 | 640 | 长输入高并发 | +| 128 | 2048 | 512 | 2560 | 长输入极限并发 | +| 1 | 4096 | 1024 | 20 | 超长输入基线 | +| 8 | 4096 | 1024 | 160 | 超长输入中并发 | +| 32 | 4096 | 1024 | 640 | 超长输入高并发 | +| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | + +## 健康检查与故障自动跳过 + +`bench_client.py` 内置了 **后台健康检查 + 请求级重试** 机制，确保某个服务挂了不会导致整个 benchmark 失败。 + +### 健康检查机制 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--health-check-interval` | 5.0 | 每 N 秒检查一次所有服务 | +| `--health-max-failures` | 2 | 连续失败 N 次才标记为不健康 | +| `--health-recovery-interval` | 10.0 | 不健康服务 N 秒后自动重试恢复 | +| `--request-max-retries` | 2 | 单个请求失败最多重试 N 次 | + +### 故障处理流程 + +``` +请求 → 负载均衡选择端口 → 发送请求 + ↓ +成功 → 记录结果 + ↓ +失败 → 换另一个端口重试（最多 --request-max-retries 次） + ↓ +全部失败 → 记录失败，继续下一个请求 +``` + +后台同时运行健康检查： +- 每 5 秒对所有端口执行 `/health` 探测 +- 连续 2 次失败的服务被标记为 **unhealthy**，不再接收新请求 +- 10 秒后自动尝试恢复，如果恢复成功则重新加入负载均衡池 + +### 输出中的故障信息 + +汇总 JSON 包含以下字段： + +```json +{ + "completed": 638, + "failed": 2, + "retried": 5, + "healthy_ports_at_end": [30005, 30006, 30007], + "per_port_stats": { + "30005": {"count": 160, "failed": 0, ...}, + "30006": {"count": 159, "failed": 1, ...}, + "30007": {"count": 160, "failed": 0, ...}, + "30008": {"count": 159, "failed": 1, ...} + } +} +``` + +- `failed`: 最终失败的请求数（重试后仍失败） +- `retried`: 至少重试过一次的请求数 +- `healthy_ports_at_end`: 压测结束时仍健康的服务端口 +- `per_port_stats[].failed`: 每个端口的失败数 + +### 场景间健康检查 + +`run_bench.sh` 在每个场景结束后会打印一次服务健康状态： + +``` +[2026-07-10T08:45:00+00:00] service health check: healthy=[30005 30006 30007 30008] unhealthy=[] +[2026-07-10T08:45:30+00:00] service health check: healthy=[30005 30006 30007] unhealthy=[30008] +[2026-07-10T08:45:30+00:00] WARNING: 1 service(s) unhealthy: 30008 +``` + +这样即使某个服务在压测中途 OOM 崩溃，其他服务仍能继续工作，benchmark 不会中断。 + +### 自定义故障参数 + +```bash +python3 bench_client.py \ + --ports 30005,30006,30007,30008 \ + --health-check-interval 3 \ + --health-max-failures 1 \ + --health-recovery-interval 5 \ + --request-max-retries 3 \ + ... +``` + +- 更敏感：间隔 3 秒、1 次失败即标记不健康、5 秒恢复 +- 更容错：单个请求最多重试 3 次 + +## 负载均衡策略 + +`bench_client.py` 支持三种负载均衡策略： + +| 策略 | 说明 | 适用场景 | +|------|------|----------| +| `round_robin` | 请求按顺序轮询分配到各端口 | **默认**，最公平，推荐 | +| `random` | 随机选择一个端口 | 简单，分布可能不均 | +| `least_conn` | 选择当前连接数最少的端口 | 需要追踪 in-flight，当前为 random 占位 | + +通过 `--lb-strategy` 参数选择： + +```bash +python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,30008 ... +``` + +## 输出格式 + +### JSONL 原始数据 (`raw_outputs/vllm_*.jsonl`) + +每行一个 JSON 对象，包含请求分发到的目标端口： + +```json +{ + "request_id": 0, + "input_tokens": 512, + "output_tokens": 256, + "first_token_time_ms": 45.23, + "e2e_latency_ms": 5234.56, + "inter_token_latencies": [18.5, 19.2, ...], + "tpot_ms": 19.8, + "mean_itl_ms": 19.5, + "success": true, + "error": "", + "target_port": 30005 +} +``` + +最后一行是汇总数据，包含 **各端口统计**： + +```json +{ + "completed": 640, + "failed": 0, + "duration": 12.34, + "request_throughput": 51.88, + "mean_ttft_ms": 48.5, + "p95_ttft_ms": 62.3, + "per_port_stats": { + "30005": {"count": 160, "mean_ttft_ms": 47.2, "p95_ttft_ms": 58.1, "mean_e2e_ms": 5200.0}, + "30006": {"count": 160, "mean_ttft_ms": 49.1, "p95_ttft_ms": 64.3, "mean_e2e_ms": 5250.0}, + "30007": {"count": 160, "mean_ttft_ms": 48.8, "p95_ttft_ms": 61.5, "mean_e2e_ms": 5230.0}, + "30008": {"count": 160, "mean_ttft_ms": 48.9, "p95_ttft_ms": 65.2, "mean_e2e_ms": 5270.0} + }, + "lb_strategy": "round_robin", + "num_services": 4 +} +``` + +### 结构化结果 (`results.json`) + +遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。 + +### 人类可读报告 (`report.md`) + +Markdown 表格，包含所有场景的 TTFT/TPOT/E2E 均值与分位数，以及 SLO 达标状态。 + +## 单独启动/停止 4 个服务 + +```bash +# 启动 4 个服务 +bash experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh + +# 停止（run_bench.sh 会自动停止，但也可以手动） +for port in 30005 30006 30007 30008; do + pid_file="experiments/dsv4_h200_vllm_tp2_custom_bench/server_port${port}.pid" + [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true + rm -f "$pid_file" +done +pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true +``` + +## 注意事项 + +- `bench_client.py` 优先使用 `aiohttp`，如果未安装则自动回退到 `urllib`（但会丢失 streaming 和精确 TTFT/ITL 测量）。 +- 默认 `temperature=0.0` 以保证输出长度稳定。 +- 输入 token 数通过空格分隔的单词数估算，实际 tokenizer 可能略有偏差。 +- 4 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 +- 每个服务占用约 2 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 +- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 +- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py new file mode 100755 index 0000000..d8edc36 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. + +Sends concurrent requests to multiple inference servers and records per-request +latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. + +Designed for TP=2 on 8-GPU setups where 4 independent services run on +(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via +round-robin or random load balancing. Unhealthy services are automatically +skipped with periodic retry. + +Usage: + # Single service (backward compatible) + python3 bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --output-file results.jsonl + + # Multi-service (4 services on ports 30005-30008) + python3 bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --lb-strategy round_robin \ + --output-file results.jsonl + +The output JSONL contains one object per request with raw timing data. +A final summary line ("completed" field) aggregates all requests. +""" + +import argparse +import asyncio +import json +import random +import sys +import time +import urllib.request +from dataclasses import asdict, dataclass, field +from typing import Any + +# Try aiohttp first; fall back to urllib for stdlib-only environments. +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + +@dataclass +class RequestResult: + request_id: int + input_tokens: int = 0 + output_tokens: int = 0 + first_token_time_ms: float = 0.0 # TTFT + e2e_latency_ms: float = 0.0 # total wall time + inter_token_latencies: list[float] = field(default_factory=list) + success: bool = False + error: str = "" + target_port: int = 0 # which service handled this request + retry_count: int = 0 # how many times this request was retried + + @property + def tpot_ms(self) -> float: + """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" + if self.output_tokens <= 1: + return 0.0 + return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) + + @property + def mean_itl_ms(self) -> float: + if not self.inter_token_latencies: + return 0.0 + return sum(self.inter_token_latencies) / len(self.inter_token_latencies) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["tpot_ms"] = self.tpot_ms + d["mean_itl_ms"] = self.mean_itl_ms + return d + + +# --------------------------------------------------------------------------- +# Health checker +# --------------------------------------------------------------------------- + +class HealthChecker: + """Periodically check service health and maintain a healthy-port list.""" + + def __init__( + self, + host: str, + ports: list[int], + check_interval: float = 5.0, + max_failures: int = 2, + recovery_interval: float = 10.0, + ): + self.host = host + self.all_ports = ports + self.check_interval = check_interval + self.max_failures = max_failures + self.recovery_interval = recovery_interval + + # port -> consecutive failure count + self._failures: dict[int, int] = {p: 0 for p in ports} + # port -> last healthy timestamp + self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} + self._lock = asyncio.Lock() + self._task: asyncio.Task | None = None + + async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: + url = f"http://{self.host}:{port}/health" + try: + if HAS_AIOHTTP and session is not None: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + return resp.status == 200 + else: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status == 200 + except Exception: + return False + + async def _check_all(self, session: aiohttp.ClientSession | None) -> None: + checks = [self._check_one(p, session) for p in self.all_ports] + results = await asyncio.gather(*checks, return_exceptions=True) + now = time.monotonic() + async with self._lock: + for port, ok in zip(self.all_ports, results): + if isinstance(ok, Exception): + ok = False + if ok: + self._failures[port] = 0 + self._last_healthy[port] = now + else: + self._failures[port] += 1 + + async def start(self, session: aiohttp.ClientSession | None) -> None: + async def _loop() -> None: + while True: + await asyncio.sleep(self.check_interval) + await self._check_all(session) + self._task = asyncio.create_task(_loop()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + @property + async def healthy_ports(self) -> list[int]: + now = time.monotonic() + async with self._lock: + healthy = [] + for p in self.all_ports: + if self._failures[p] < self.max_failures: + healthy.append(p) + elif now - self._last_healthy[p] > self.recovery_interval: + # Give it another chance after recovery_interval. + self._failures[p] = max(0, self._failures[p] - 1) + healthy.append(p) + return healthy + + async def is_healthy(self, port: int) -> bool: + ports = await self.healthy_ports + return port in ports + + +# --------------------------------------------------------------------------- +# Load balancer with failover +# --------------------------------------------------------------------------- + +class LoadBalancer: + """Distribute requests across healthy backend ports with auto-failover.""" + + def __init__( + self, + ports: list[int], + health_checker: HealthChecker, + strategy: str = "round_robin", + ): + self.all_ports = ports + self.health_checker = health_checker + self.strategy = strategy + self._idx = 0 + self._lock = asyncio.Lock() + + def _next_round_robin(self, healthy: list[int]) -> int: + if not healthy: + return self.all_ports[self._idx % len(self.all_ports)] + for _ in range(len(self.all_ports)): + port = self.all_ports[self._idx % len(self.all_ports)] + self._idx += 1 + if port in healthy: + return port + # Fallback: all unhealthy, pick first healthy anyway. + return healthy[0] if healthy else self.all_ports[0] + + def _next_random(self, healthy: list[int]) -> int: + if healthy: + return random.choice(healthy) + return random.choice(self.all_ports) + + async def next_port(self) -> int: + healthy = await self.health_checker.healthy_ports + async with self._lock: + if self.strategy == "round_robin": + return self._next_round_robin(healthy) + elif self.strategy in ("random", "least_conn"): + return self._next_random(healthy) + else: + return self._next_round_robin(healthy) + + @property + def port_count(self) -> int: + return len(self.all_ports) + + +# --------------------------------------------------------------------------- +# Token helpers +# --------------------------------------------------------------------------- + +def make_prompt(token_len: int, vocab_size: int = 32000) -> str: + """Generate a random-ish prompt of approximately `token_len` tokens.""" + words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", + "dog", "hello", "world", "deep", "seek", "model", "test", + "benchmark", "performance", "latency", "throughput", "token"] + needed = max(token_len, 1) + tokens: list[str] = [] + while len(tokens) < needed: + tokens.extend(words) + return " ".join(tokens[:needed]) + + +# --------------------------------------------------------------------------- +# aiohttp-based async request (preferred) +# --------------------------------------------------------------------------- + +async def send_request_aiohttp( + session: aiohttp.ClientSession, + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + headers = {"Content-Type": "application/json"} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + first_token_received = False + last_token_time = 0.0 + + try: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + text = await resp.text() + result.error = f"HTTP {resp.status}: {text[:200]}" + return result + + async for line in resp.content: + line = line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data_str = line[len("data: "):] + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + content = delta.get("content", "") + if content: + now = time.perf_counter() + if not first_token_received: + result.first_token_time_ms = (now - start_time) * 1000 + first_token_received = True + else: + result.inter_token_latencies.append((now - last_token_time) * 1000) + last_token_time = now + result.output_tokens += 1 + + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.input_tokens = len(prompt.split()) # approximate + result.success = True + + except asyncio.TimeoutError: + result.error = "timeout" + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# urllib-based synchronous request (fallback, no aiohttp) +# --------------------------------------------------------------------------- + +def send_request_urllib( + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + } + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=600) as resp: + body = json.loads(resp.read().decode("utf-8")) + + choices = body.get("choices", []) + if not choices: + result.error = "no choices in response" + return result + + content = choices[0].get("message", {}).get("content", "") + result.output_tokens = len(content.split()) if content else 0 + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.first_token_time_ms = result.e2e_latency_ms + result.input_tokens = len(prompt.split()) + result.success = True + + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# Unified send_request wrapper with retry & failover +# --------------------------------------------------------------------------- + +async def send_request_with_retry( + session: aiohttp.ClientSession | None, + host: str, + lb: LoadBalancer, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, + max_retries: int = 2, +) -> RequestResult: + """Send a request, retrying on failure with a different port each time.""" + result = RequestResult(request_id=request_id) + for attempt in range(max_retries + 1): + port = await lb.next_port() + if HAS_AIOHTTP and session is not None: + result = await send_request_aiohttp( + session, host, port, model, request_id, prompt, max_tokens, temperature + ) + else: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, send_request_urllib, + host, port, model, request_id, prompt, max_tokens, temperature + ) + result.retry_count = attempt + if result.success: + return result + # If failed, try another port on next iteration (unless last attempt). + if attempt < max_retries: + await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish + return result + + +# --------------------------------------------------------------------------- +# Benchmark orchestration +# --------------------------------------------------------------------------- + +async def run_benchmark( + host: str, + ports: list[int], + lb: LoadBalancer, + health_checker: HealthChecker, + model: str, + concurrency: int, + input_len: int, + output_len: int, + num_prompts: int, + temperature: float, + output_file: str, +) -> dict[str, Any]: + """Run the benchmark and write a JSONL file.""" + prompts = [make_prompt(input_len) for _ in range(num_prompts)] + results: list[RequestResult] = [] + semaphore = asyncio.Semaphore(concurrency) + + start = time.perf_counter() + + if HAS_AIOHTTP: + connector = aiohttp.TCPConnector(limit=concurrency * 2) + timeout = aiohttp.ClientTimeout(total=600, sock_read=300) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + await health_checker.start(session) + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + session, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + else: + await health_checker.start(None) + from concurrent.futures import ThreadPoolExecutor + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + None, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + + duration = time.perf_counter() - start + + # Write JSONL + with open(output_file, "w", encoding="utf-8") as f: + for r in results: + f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") + + # Compute summary + successful = [r for r in results if r.success] + failed = len(results) - len(successful) + retried = [r for r in results if r.retry_count > 0] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + ttfts = [r.first_token_time_ms for r in successful] + tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] + e2es = [r.e2e_latency_ms for r in successful] + itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] + + total_input_tokens = sum(r.input_tokens for r in successful) + total_output_tokens = sum(r.output_tokens for r in successful) + + # Per-port breakdown + per_port_stats: dict[int, dict[str, Any]] = {} + for port in ports: + port_results = [r for r in successful if r.target_port == port] + port_failed = [r for r in results if r.target_port == port and not r.success] + if port_results or port_failed: + port_ttfts = [r.first_token_time_ms for r in port_results] + port_e2es = [r.e2e_latency_ms for r in port_results] + per_port_stats[port] = { + "count": len(port_results), + "failed": len(port_failed), + "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, + "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, + "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, + } + + # Health-check stats + healthy_at_end = await health_checker.healthy_ports + + summary = { + "completed": len(successful), + "failed": failed, + "retried": len(retried), + "duration": duration, + "request_throughput": len(successful) / duration if duration > 0 else 0.0, + "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, + "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, + "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, + "median_ttft_ms": percentile(ttfts, 50), + "p90_ttft_ms": percentile(ttfts, 90), + "p95_ttft_ms": percentile(ttfts, 95), + "p99_ttft_ms": percentile(ttfts, 99), + "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, + "median_tpot_ms": percentile(tpots, 50), + "p90_tpot_ms": percentile(tpots, 90), + "p95_tpot_ms": percentile(tpots, 95), + "p99_tpot_ms": percentile(tpots, 99), + "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, + "median_e2e_latency_ms": percentile(e2es, 50), + "p90_e2e_latency_ms": percentile(e2es, 90), + "p95_e2e_latency_ms": percentile(e2es, 95), + "p99_e2e_latency_ms": percentile(e2es, 99), + "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, + "median_itl_ms": percentile(itls, 50), + "p90_itl_ms": percentile(itls, 90), + "p95_itl_ms": percentile(itls, 95), + "p99_itl_ms": percentile(itls, 99), + "per_port_stats": per_port_stats, + "healthy_ports_at_end": healthy_at_end, + "lb_strategy": lb.strategy, + "num_services": lb.port_count, + } + + # Append summary as the last line + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + + return summary + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30005, + help="Single service port (backward compatible)") + parser.add_argument("--ports", default="", + help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") + parser.add_argument("--model", default="deepseek-v4-flash") + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--input-len", type=int, default=512) + parser.add_argument("--output-len", type=int, default=256) + parser.add_argument("--num-prompts", type=int, default=10) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--lb-strategy", default="round_robin", + choices=["round_robin", "random", "least_conn"], + help="Load balancing strategy across services") + parser.add_argument("--health-check-interval", type=float, default=5.0, + help="Seconds between health checks (default: 5)") + parser.add_argument("--health-max-failures", type=int, default=2, + help="Consecutive failures before marking a port unhealthy (default: 2)") + parser.add_argument("--health-recovery-interval", type=float, default=10.0, + help="Seconds before retrying an unhealthy port (default: 10)") + parser.add_argument("--request-max-retries", type=int, default=2, + help="Max retries per request on failure (default: 2)") + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + + # Determine ports: --ports takes precedence, else fall back to --port. + if args.ports: + ports = [int(p.strip()) for p in args.ports.split(",")] + else: + ports = [args.port] + + health_checker = HealthChecker( + host=args.host, + ports=ports, + check_interval=args.health_check_interval, + max_failures=args.health_max_failures, + recovery_interval=args.health_recovery_interval, + ) + lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) + + summary = asyncio.run(run_benchmark( + host=args.host, + ports=ports, + lb=lb, + health_checker=health_checker, + model=args.model, + concurrency=args.concurrency, + input_len=args.input_len, + output_len=args.output_len, + num_prompts=args.num_prompts, + temperature=args.temperature, + output_file=args.output_file, + )) + + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env new file mode 100644 index 0000000..84a30c5 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env @@ -0,0 +1,75 @@ +# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench +# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. +# +# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: +# Service 1: port 30005, GPUs 0,1 +# Service 2: port 30006, GPUs 2,3 +# Service 3: port 30007, GPUs 4,5 +# Service 4: port 30008, GPUs 6,7 +# +# bench_client.py distributes requests across all 4 services via round-robin +# load balancing, giving accurate cluster-wide throughput even at low +# per-service concurrency. + +EXPERIMENT="dsv4_h200_vllm_tp2_custom_bench" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Primary port (backward compatible); multi-service ports are in PORTS. +PORT="30005" +PORTS="30005,30006,30007,30008" + +BACKEND="vllm" +ENGINE="vllm" + +# Native virtual environments on the host. +VENV_SERVER="/data/user1/yy/envs/vllm" +VENV_CLIENT="/data/user1/yy/envs/vllm" + +# Load balancing strategy: round_robin | random | least_conn +LB_STRATEGY="round_robin" + +# Benchmark scenarios: "concurrency input_len output_len num_prompts". +# For multi-service we include low-concurrency scenarios to show how +# requests spread across 4 services. +SCENARIOS=( + "1 512 256 20" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + "1 2048 512 20" + "8 2048 512 160" + "32 2048 512 640" + "128 2048 512 2560" + "1 4096 1024 20" + "8 4096 1024 160" + "32 4096 1024 640" + "128 4096 1024 2560" + "1 8192 1024 20" + "8 8192 1024 160" + "32 8192 1024 640" + "128 8192 1024 2560" + "1 16384 1024 20" + "8 16384 1024 160" + "32 16384 1024 640" + "128 16384 1024 2560" + "1 32768 1024 20" + "8 32768 1024 160" + "32 32768 1024 640" + "128 32768 1024 2560" + "1 65536 1024 20" + "8 65536 1024 160" + "32 65536 1024 640" + "128 65536 1024 2560" + "1 131072 1024 20" + "8 131072 1024 160" + "32 131072 1024 640" + "128 131072 1024 2560" +) + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py b/experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py new file mode 100755 index 0000000..eea1a2e --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. + +Reads JSONL files produced by bench_client.py (one request per line, +last line is the summary) and generates: + - results.json (appended scenarios, following the standard schema) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: + """Read the summary JSON (last line) and all per-request lines.""" + requests: list[dict] = [] + summary: dict | None = None + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The summary line has "completed" and "duration" keys. + if "completed" in obj and "duration" in obj: + summary = obj + else: + requests.append(obj) + return summary, requests + + +def compute_metrics(summary: dict, requests: list[dict]) -> dict: + completed = summary.get("completed", 0) + total = len(requests) + failed = total - completed if total > 0 else 0 + duration_s = summary.get("duration", 0.0) + + # Extract per-request metrics for percentiles. + ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] + tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] + e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] + itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": summary.get("request_throughput", 0.0), + "input_token_throughput": summary.get("input_throughput", 0.0), + "output_token_throughput": summary.get("output_throughput", 0.0), + "total_token_throughput": summary.get("total_throughput", 0.0), + "total_input_tokens": summary.get("total_input_tokens", 0), + "total_output_tokens": summary.get("total_output_tokens", 0), + "e2e_ms": { + "mean": summary.get("mean_e2e_latency_ms", 0.0), + "p50": percentile(e2es, 50), + "p90": percentile(e2es, 90), + "p95": percentile(e2es, 95), + "p99": percentile(e2es, 99), + }, + "ttft_ms": { + "mean": summary.get("mean_ttft_ms", 0.0), + "p50": percentile(ttfts, 50), + "p90": percentile(ttfts, 90), + "p95": percentile(ttfts, 95), + "p99": percentile(ttfts, 99), + }, + "tpot_ms": { + "mean": summary.get("mean_tpot_ms", 0.0), + "p50": percentile(tpots, 50), + "p90": percentile(tpots, 90), + "p95": percentile(tpots, 95), + "p99": percentile(tpots, 99), + }, + "itl_ms": { + "mean": summary.get("mean_itl_ms", 0.0), + "p50": percentile(itls, 50), + "p90": percentile(itls, 90), + "p95": percentile(itls, 95), + "p99": percentile(itls, 99), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") + f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") + f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") + f.write(f"- **Result root**: `{result_root}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + summary, requests = parse_jsonl(jsonl_path) + if summary is None: + continue + + metrics = compute_metrics(summary, requests) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh new file mode 100755 index 0000000..217473d --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# Custom benchmark orchestrator for multi-service vLLM TP=2 on 8x H200. +# +# This runs 4 independent vLLM services (TP=2 each) on GPU pairs (0,1),(2,3),(4,5),(6,7). +# bench_client.py distributes requests across all 4 services via round-robin LB, +# giving accurate cluster-wide throughput even at low per-service concurrency. +# +# Usage: +# bash run_bench.sh +# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "services=${PORTS}" +log "lb_strategy=${LB_STRATEGY}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json, sys +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", "4 512 256 80", "8 512 256 160", + "16 512 256 320", "32 512 256 640", "64 512 256 1280", "128 512 256 2560", + "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", + "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# --------------------------------------------------------------------------- +# Server helpers (multi-service) +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +are_all_services_healthy() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + for p in "${PORT_LIST[@]}"; do + if ! is_server_healthy "$p"; then + return 1 + fi + done + return 0 +} + +stop_all_servers() { + log "stopping all vllm services" + # Kill by PID files first. + for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do + [[ -f "$pid_file" ]] || continue + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "killing pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + done + # Fallback: kill any remaining vllm processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + sleep 2 +} + +start_all_servers() { + log "starting all vllm services with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + + if ! are_all_services_healthy "$PORTS"; then + log "error: not all vllm services became healthy" + exit 1 + fi + log "all vllm services are healthy" +} + +# --------------------------------------------------------------------------- +# Pre-benchmark health check with detailed reporting +# --------------------------------------------------------------------------- + +check_services_detailed() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy_count=0 + local unhealthy_ports="" + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + ((healthy_count++)) + else + unhealthy_ports="${unhealthy_ports}${p} " + fi + done + echo "$healthy_count" + if [[ -n "$unhealthy_ports" ]]; then + echo "unhealthy: ${unhealthy_ports}" >&2 + fi +} + +wait_for_all_services() { + local ports="$1" + local max_wait="${2:-240}" + local interval="${3:-5}" + local elapsed=0 + while (( elapsed < max_wait )); do + local healthy_count + healthy_count="$(check_services_detailed "$ports" | head -1)" + IFS=',' read -ra PORT_LIST <<< "$ports" + if (( healthy_count == ${#PORT_LIST[@]} )); then + return 0 + fi + log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" + sleep "$interval" + ((elapsed += interval)) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Mid-benchmark health check (called between scenarios) +# --------------------------------------------------------------------------- + +check_and_report_services() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy=() + local unhealthy=() + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + healthy+=("$p") + else + unhealthy+=("$p") + fi + done + log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" + if [[ ${#unhealthy[@]} -gt 0 ]]; then + log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" + fi + return ${#unhealthy[@]} +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_all_servers + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming services are already running" + if ! wait_for_all_services "$PORTS"; then + log "error: not all healthy services found on ports ${PORTS}" + exit 1 + fi +else + stop_all_servers + start_all_servers +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi + + output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" + + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ + --host 127.0.0.1 \ + --ports "$PORTS" \ + --model "$SERVED_MODEL_NAME" \ + --concurrency "$concurrency" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --lb-strategy "$LB_STRATEGY" \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file "$output_file" \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + # After failure, check service health before continuing. + check_and_report_services "$PORTS" + continue + } + + log "finished scenario: output=${output_file}" + + # Between scenarios, report service health so we can spot degradation early. + check_and_report_services "$PORTS" || true +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh new file mode 100755 index 0000000..cc3b022 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Start 4 independent vLLM TP=2 services on 8x H200. +# Each service runs on a pair of GPUs: (0,1), (2,3), (4,5), (6,7). +# This maximizes GPU utilization for low-concurrency workloads. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_SERVER}" +export PATH="$VENV/bin:$PATH" +VLLM="$VENV/bin/vllm" + +TP=2 +LOG_DIR="${SCRIPT_DIR}/logs" +mkdir -p "$LOG_DIR" + +# 4 services, each on 2 GPUs. +# Port and GPU pair mapping. +SERVICES=( + "30005:0,1" + "30006:2,3" + "30007:4,5" + "30008:6,7" +) + +# Kill any existing vLLM processes for this model. +pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true +sleep 2 + +# Remove old PID files. +rm -f "${SCRIPT_DIR}"/*.pid + +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + GPUS="${svc##*:}" + LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + + echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" + echo "Log: ${LOG}" + + CUDA_VISIBLE_DEVICES="${GPUS}" \ + nohup "$VLLM" serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --block-size 256 \ + --port "$PORT" \ + > "$LOG" 2>&1 & + PID=$! + echo $PID > "$PID_FILE" + echo "PID: $PID" +done + +echo "" +echo "Waiting for all 4 services to become healthy..." + +MAX_WAIT=240 +all_healthy=0 +for i in $(seq 1 $MAX_WAIT); do + all_healthy=1 + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + all_healthy=0 + break + fi + done + + if [[ "$all_healthy" -eq 1 ]]; then + echo "All 4 services are healthy!" + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + echo " - http://127.0.0.1:${PORT}" + done + exit 0 + fi + + # Check if any process died early. + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + if [[ -f "$PID_FILE" ]]; then + PID="$(cat "$PID_FILE")" + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + tail -200 $LOG 2>/dev/null || true + exit 1 + fi + fi + done + + if (( i % 10 == 0 )); then + echo "Waiting... (${i}/${MAX_WAIT})" + fi + sleep 5 +done + +echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + echo "--- Last 50 lines of port ${PORT} ---" + tail -50 $LOG 2>/dev/null || true +done +exit 1 From a388358b37d68f9a1bf2143b0ed069c0a0926f26 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 07:05:36 +0000 Subject: [PATCH 35/53] fix(sglang-tp-dp-matrix): export DOCKER_IMAGE/USE_DOCKER_CLIENT and add persistent container support - Fix docker invalid reference format by exporting DOCKER_IMAGE and USE_DOCKER_CLIENT to bash subshells (run_bench_serving) - Add persistent container mode: start_sglang_container.sh + run_sglang_in_container.sh for faster config switching - Add README.md with setup and migration guide --- .../dsv4_h200_sglang_tp_dp_matrix/README.md | 234 ++++++++++++++++++ .../run_bench.sh | 41 ++- .../run_sglang_in_container.sh | 93 +++++++ .../start_sglang_container.sh | 70 ++++++ 4 files changed, 426 insertions(+), 12 deletions(-) create mode 100644 experiments/dsv4_h200_sglang_tp_dp_matrix/README.md create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/run_sglang_in_container.sh create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_container.sh diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/README.md b/experiments/dsv4_h200_sglang_tp_dp_matrix/README.md new file mode 100644 index 0000000..a95a847 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/README.md @@ -0,0 +1,234 @@ +# SGLang TP×DP Matrix Benchmark + +Benchmark SGLang inference performance across different Tensor Parallelism (TP) × Data Parallelism (DP) configurations for large MoE models (e.g., DeepSeek-V4-Flash). + +## What This Experiment Does + +This experiment systematically tests how different parallel strategies affect serving performance: + +| Config | TP | DP | GPUs per replica | Replicas | +|--------|----|----|------------------|----------| +| tp2_dp4 | 2 | 4 | 2 | 4 | +| tp4_dp2 | 4 | 2 | 4 | 2 | +| tp8_dp1 | 8 | 1 | 8 | 1 | + +For each configuration, it runs a matrix of input/output sequence lengths and concurrency levels, measuring: +- **TTFT** (Time To First Token) +- **TPOT** (Time Per Output Token) +- **Throughput** (requests/s, output tokens/s) +- **GPU memory usage** +- **SLO compliance** (TTFT P95 < 3000ms, TPOT mean < 50ms) + +## Quick Start + +### 1. Prerequisites + +- NVIDIA GPU server with Docker and NVIDIA Container Toolkit +- Model weights mounted locally (e.g., `/data/models/DeepSeek-V4-Flash`) +- ShareGPT-format dataset JSON for the random sampler (e.g., `/data/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json`) +- SGLang Docker image pulled locally + +### 2. Configure the Experiment + +Edit `config.env` to match your environment: + +```bash +# Model settings +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Docker image (change this if you use a custom or private registry image) +DOCKER_IMAGE="lmsysorg/sglang:latest" + +# Dataset path (change this to your local dataset) +DATASET_PATH="/data/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json" + +# GPU devices to use +CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" + +# TP×DP configurations to test +PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) +``` + +### 3. Run the Experiment + +#### Option A: Persistent Container Mode (Recommended) + +Starts a long-running container and reuses it across configurations. This avoids reloading the model and preserves DeepGEMM JIT cache, saving significant time. + +```bash +# Step 1: Start the persistent container (run once) +bash start_sglang_container.sh + +# Step 2: Run the benchmark +CONTAINER_NAME="dsv4_h200_sglang_tp_dp_matrix_sglang_container" \ + bash run_bench.sh +``` + +#### Option B: One-Container-Per-Config Mode + +Each TP×DP configuration starts a fresh Docker container. Slower but simpler. + +```bash +# Just run the main script +bash run_bench.sh +``` + +### 4. View Results + +Results are saved under `results//`: + +``` +results/ +└── full_20260710-063109/ + ├── tp2_dp4/ + │ ├── results.json # Parsed metrics per scenario + │ ├── raw_outputs/ # Raw benchmark output files + │ ├── gpu_logs/ # GPU memory CSV logs + │ └── logs/ # Detailed logs per scenario + ├── tp4_dp2/ + │ └── ... + ├── tp8_dp1/ + │ └── ... + ├── comparison.md # Cross-config comparison report + └── logs/ + └── orchestrator.log # Main experiment log +``` + +Generate the comparison report manually: + +```bash +python3 compare.py --run-root results/ --output results//comparison.md +``` + +## Configuration Reference + +### `config.env` — Main Configuration File + +| Variable | Default | Description | +|----------|---------|-------------| +| `MODEL_NAME` | `DeepSeek-V4-Flash` | Human-readable model name | +| `MODEL_PATH` | `/data/models/DeepSeek-V4-Flash` | **Local path to model weights** | +| `SERVED_MODEL_NAME` | `deepseek-v4-flash` | Model name exposed by the server | +| `DOCKER_IMAGE` | `lmsysorg/sglang:latest` | **SGLang Docker image** | +| `DATASET_PATH` | `/data/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json` | **Local dataset for random sampler** | +| `CUDA_VISIBLE_DEVICES` | `0,1,2,3,4,5,6,7` | GPU devices to use | +| `PARALLEL_CONFIGS` | `("2 4" "4 2" "8 1")` | TP×DP configurations to test | +| `CONTEXT_LENGTH` | `1048576` | Max context length (tokens) | +| `MAX_RUNNING_REQUESTS` | `128` | Max concurrent requests | +| `MEM_FRACTION_STATIC` | `0.88` | Static memory fraction | +| `MOE_RUNNER_BACKEND` | `marlin` | MoE backend (marlin, flashinfer, etc.) | +| `USE_DOCKER` | `1` | Run server in Docker (1) or native venv (0) | +| `USE_DOCKER_CLIENT` | `1` | Run benchmark client in Docker (1) or native (0) | +| `SGLANG_PORT` | `30031` | Server port | +| `MATRIX_FILE` | `matrix.json` | Scenario matrix definition | +| `MATRIX_MODE` | `Y` | Test only Y (mandatory), or Y+P (include optional) | +| `CONCURRENCY_SAMPLES` | `2` | Concurrency sampling density | +| `SCENARIO_TIMEOUT_S` | `1800` | Per-scenario timeout (seconds) | +| `DRY_RUN` | `0` | If 1, only print plan without running | +| `GRID_LIMIT` | `0` | Limit scenarios per config (0 = all) | + +### `matrix.json` — Test Scenario Matrix + +Defines which (Input Sequence Length, Output Sequence Length) combinations to test: + +```json +{ + "mode": "Y", + "matrix": { + "1024": { "128": "Y", "256": "Y", ... }, + "4096": { "128": "Y", "256": "Y", ... } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 } + } +} +``` + +Mark meanings: +- **Y** = Must test; failure stops the config +- **P** = Optional; failure is recorded but continues +- **N** = Skip + +### `smoke_matrix.json` — Quick Verification + +Small matrix for fast testing. Use it to verify the setup works: + +```bash +MATRIX_FILE="${PWD}/smoke_matrix.json" \ + GRID_LIMIT=4 \ + bash run_bench.sh +``` + +## Script Reference + +| Script | Purpose | When to Run | +|--------|---------|-------------| +| `run_bench.sh` | **Main experiment orchestrator** | Always run this | +| `start_sglang_container.sh` | Start a persistent Docker container | Before run_bench.sh if using persistent mode | +| `run_sglang_in_container.sh` | Start/restart server inside persistent container | Called by run_bench.sh | +| `start_sglang_dp.sh` | Entry point for starting server | Called by run_bench.sh | +| `start_sglang_docker.sh` | Start a fresh Docker container per config | Called by start_sglang_dp.sh | +| `generate_scenarios.py` | Generate scenario TSV from matrix.json | Called by run_bench.sh | +| `compare.py` | Generate cross-config comparison report | Called by run_bench.sh, or run manually | + +## Migrating to a New Server + +When moving this experiment to a different server, update these fields in `config.env`: + +1. **Model path**: `MODEL_PATH` → your local model weights directory +2. **Docker image**: `DOCKER_IMAGE` → your SGLang image (e.g., `myregistry/sglang:v0.4.0`) +3. **Dataset path**: `DATASET_PATH` → your local ShareGPT-format JSON +4. **GPU devices**: `CUDA_VISIBLE_DEVICES` → match your hardware +5. **TP×DP configs**: `PARALLEL_CONFIGS` → adjust to your GPU count + - Total GPUs must equal `TP × DP` for each config + - Example: 8 GPUs → `(2 4)`, `(4 2)`, `(8 1)` are all valid + +Optional adjustments: +- `CONTEXT_LENGTH` → model's max context length +- `MEM_FRACTION_STATIC` → adjust if you hit OOM +- `MOE_RUNNER_BACKEND` → `marlin`, `flashinfer`, etc. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `docker: invalid reference format` | `DOCKER_IMAGE` not exported to subshell | Already fixed; ensure `run_bench.sh` exports it | +| Server health check timeout | Model loading takes >20 min | Increase wait loops in `start_sglang_docker.sh` or use persistent container mode | +| `CUDA out of memory` | Config exceeds GPU memory | Reduce `MEM_FRACTION_STATIC` or test smaller ISL/DSL | +| Benchmark client can't reach server | Network isolation | Ensure both server and client use `--network host` | +| Stale container from previous run | Container name collision | `docker rm -f ` before restarting | + +## Monitoring a Running Experiment + +```bash +# Watch the main log +tail -f /tmp/full_experiment.log + +# Check if the orchestrator is still running +ps aux | grep run_bench | grep -v grep + +# Check GPU utilization +watch -n 1 nvidia-smi + +# Check server health +curl http://127.0.0.1:30031/health +``` + +## Output Files + +- `results///results.json` — Parsed metrics per scenario +- `results///raw_outputs/` — Raw `sglang.bench_serving` JSONL output +- `results///gpu_logs/` — GPU memory usage CSV +- `results//comparison.md` — Side-by-side comparison across all configs +- `results//skipped_after_oom.csv` — Record of OOM/failed scenarios + +## License + +Follows the project's existing license. Model weights and Docker images are subject to their respective licenses. diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index a1bc182..53d0fbc 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -12,9 +12,6 @@ source "${SCRIPT_DIR}/../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -# Export variables used inside functions that are called via bash -c subshells. -export DATASET_PATH MODEL_PATH RESULT_BASE - RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" @@ -24,6 +21,9 @@ GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" DRY_RUN="${DRY_RUN:-0}" GRID_LIMIT="${GRID_LIMIT:-0}" +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT + if [[ -x "${VENV_CLIENT}/bin/python" ]]; then PYTHON="${VENV_CLIENT}/bin/python" else @@ -55,17 +55,29 @@ stop_server() { if [[ -f "$pid_file" ]]; then local pid pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})" - kill "$pid" 2>/dev/null || true - sleep 5 - kill -9 "$pid" 2>/dev/null || true + # In container-reuse mode, pid is the server PID inside the container. + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server inside container (tp=${tp}, dp=${dp}, pid=${pid})" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + else + if kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi fi rm -f "$pid_file" fi - # Fallback: remove any Docker container started by this experiment. - docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + # Fallback: remove any Docker container started by this experiment (legacy mode). + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi # Fallback: kill any sglang serve processes for this model. pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true @@ -101,8 +113,13 @@ start_server() { local dp="$2" log "starting sglang server tp=${tp} dp=${dp}" - bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \ - >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + fi if ! is_server_healthy; then log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_sglang_in_container.sh new file mode 100755 index 0000000..69a08b0 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Run SGLang server inside the already-running benchmark container. +# Usage: run_sglang_in_container.sh +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" + +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_in_container_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Build server args. +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server inside container (TP=${TP}, DP=${DP}) ===" +echo "Container: $NAME" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Kill any existing sglang process inside container first. +docker exec "$NAME" pkill -9 -f "sglang serve" 2>/dev/null || true +sleep 3 + +# Start sglang serve inside the container in background. +# We use nohup so it survives after docker exec returns. +docker exec -d \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$NAME" \ + bash -c "nohup sglang ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &" + +# Wait for the server process to appear inside container. +sleep 2 +SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang serve" | head -n 1 || true) +if [[ -z "$SERVER_PID" ]]; then + echo "ERROR: sglang server process not found inside container" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -50 || true + exit 1 +fi + +echo "Server PID inside container: $SERVER_PID" +echo "$SERVER_PID" > "$PID_FILE" + +echo "Waiting for health on port ${PORT}..." +for i in $(seq 1 360); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + # Check if server process is still alive. + if ! docker exec "$NAME" kill -0 "$SERVER_PID" 2>/dev/null; then + echo "ERROR: SGLang server exited early" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true + exit 1 + fi + echo "Waiting... ($i/360)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 360 retries (30 mins)" +docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true +exit 1 diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_container.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_container.sh new file mode 100755 index 0000000..2cf469c --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_container.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Start a long-running Docker container for SGLang benchmarking. +# The container stays alive (sleep infinity) so we can docker exec into it +# to run different TP×DP configurations without losing DeepGEMM JIT cache. +# +# Usage: start_sglang_container.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_container.pid" + +# Persistent cache directory on host for DeepGEMM JIT kernels. +CACHE_DIR="${RUNTIME_BASE}/cache" +mkdir -p "${CACHE_DIR}/deep_gemm" "${CACHE_DIR}/tvm-ffi" + +# Clean up any stale container. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_container_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +echo "=== Starting SGLang benchmark container ===" +echo "Image: $IMAGE" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Cache dir: $CACHE_DIR" +echo "Log: $LOG" + +# Start a long-running container. +# We override the entrypoint to sleep infinity so the container stays alive. +docker run -d \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint /bin/bash \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}/deep_gemm:/root/.cache/deep_gemm" \ + -v "${CACHE_DIR}/tvm-ffi:/root/.cache/tvm-ffi" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + -c "sleep infinity" \ + > "$LOG" 2>&1 + +# Wait a moment for container to be ready. +sleep 2 + +if ! docker ps --filter "name=$NAME" --format '{{.Names}}' | grep -q "$NAME"; then + echo "ERROR: Container failed to start" + tail -50 "$LOG" + exit 1 +fi + +# Record the container ID for later use. +docker inspect -f '{{.Id}}' "$NAME" > "$PID_FILE" + +echo "Container $NAME is running" +echo "Log: $LOG" From b5d409ea3d627399f956b9ae6ee60adcc6d8036c Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 09:19:41 +0000 Subject: [PATCH 36/53] chore: add experiments/*/runtime/ to .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dd1d048..5758a51 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ __pycache__/ # 中间日志 logs/ *.log - +*.csv # 数据集（可重新下载） datasets/ @@ -30,5 +30,8 @@ experiments/*/results/*/logs/ experiments/*/results/*/*/raw_outputs/ experiments/*/results/*/*/logs/ +# 运行时目录（pid 文件、缓存、临时文件） +experiments/*/runtime/ + # 无关项目 loomeval_yy/ From cd539c4d7dad4aeb06c2690235d008e26ee6eeea Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 09:20:41 +0000 Subject: [PATCH 37/53] chore: add tsv files to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5758a51..8173ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ experiments/*/results/*/raw_outputs/ experiments/*/results/*/logs/ experiments/*/results/*/*/raw_outputs/ experiments/*/results/*/*/logs/ +experiments/*/results/*/*.tsv +experiments/*/results/*/*/*.tsv # 运行时目录（pid 文件、缓存、临时文件） experiments/*/runtime/ From 4d5b6a77ca419c26a09e21936f8af9e319925b23 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Fri, 10 Jul 2026 09:40:10 +0000 Subject: [PATCH 38/53] Add custom benchmark setup for vLLM TP=8 on dsv4 H200 - Create configuration file for the custom benchmark experiment. - Implement result parsing script to handle JSONL outputs from the benchmark. - Develop run script to orchestrate the benchmark execution, including server management and health checks. - Add server start script to launch a single vLLM service on all available GPUs. --- .../config.env | 9 + .../results/20260710-091141/results.json | 47 ++ .../results/20260710-091522/results.json | 47 ++ .../results/20260710-091700/results.json | 47 ++ .../start_server.sh | 1 + .../README.md | 273 ++++++++ .../bench_client.py | 617 ++++++++++++++++++ .../config.env | 84 +++ .../parse_results.py | 206 ++++++ .../results/20260710-091141/results.json | 47 ++ .../results/20260710-091522/results.json | 47 ++ .../results/20260710-091700/results.json | 47 ++ .../run_bench.sh | 277 ++++++++ .../start_server.sh | 109 ++++ .../dsv4_h200_vllm_tp4_custom_bench/README.md | 124 ++++ .../bench_client.py | 617 ++++++++++++++++++ .../config.env | 84 +++ .../parse_results.py | 206 ++++++ .../run_bench.sh | 278 ++++++++ .../start_server.sh | 108 +++ .../README.md | 126 ++++ .../bench_client.py | 617 ++++++++++++++++++ .../config.env | 84 +++ .../parse_results.py | 206 ++++++ .../run_bench.sh | 278 ++++++++ .../start_server.sh | 107 +++ .../dsv4_h200_vllm_tp8_custom_bench/README.md | 101 +++ .../bench_client.py | 617 ++++++++++++++++++ .../config.env | 83 +++ .../parse_results.py | 206 ++++++ .../run_bench.sh | 272 ++++++++ .../start_server.sh | 107 +++ .../README.md | 103 +++ .../bench_client.py | 617 ++++++++++++++++++ .../config.env | 83 +++ .../parse_results.py | 206 ++++++ .../run_bench.sh | 272 ++++++++ .../start_server.sh | 106 +++ 38 files changed, 7466 insertions(+) create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json create mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench/README.md create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench/config.env create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench/README.md create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench/config.env create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh create mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env index 84a30c5..2b2d56c 100644 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env @@ -35,6 +35,7 @@ LB_STRATEGY="round_robin" # requests spread across 4 services. SCENARIOS=( "1 512 256 20" + "2 512 256 40" "4 512 256 80" "8 512 256 160" "16 512 256 320" @@ -69,6 +70,14 @@ SCENARIOS=( "8 131072 1024 160" "32 131072 1024 640" "128 131072 1024 2560" + "1 262144 1024 20" + "8 262144 1024 160" + "32 262144 1024 640" + "128 262144 1024 2560" + "1 524288 1024 20" + "8 524288 1024 160" + "32 524288 1024 640" + "128 524288 1024 2560" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json new file mode 100644 index 0000000..64dd782 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json @@ -0,0 +1,47 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp2_custom_bench", + "run_id": "20260710-091141", + "timestamp": "2026-07-10T09:11:41+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "39fb076", + "git_dirty": "dirty", + "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json new file mode 100644 index 0000000..3196c06 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json @@ -0,0 +1,47 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp2_custom_bench", + "run_id": "20260710-091522", + "timestamp": "2026-07-10T09:15:22+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "39fb076", + "git_dirty": "dirty", + "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json new file mode 100644 index 0000000..16c1892 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json @@ -0,0 +1,47 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp2_custom_bench", + "run_id": "20260710-091700", + "timestamp": "2026-07-10T09:17:00+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "39fb076", + "git_dirty": "dirty", + "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh index cc3b022..99c85f4 100755 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh @@ -47,6 +47,7 @@ for svc in "${SERVICES[@]}"; do --tensor-parallel-size "$TP" \ --kv-cache-dtype fp8 \ --block-size 256 \ + --served-model-name "$SERVED_MODEL_NAME" \ --port "$PORT" \ > "$LOG" 2>&1 & PID=$! diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md new file mode 100644 index 0000000..dae2993 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md @@ -0,0 +1,273 @@ +# dsv4_h200_vllm_tp2_custom_bench_no_fp8 + +> **Note:** This variant does **NOT** use `--kv-cache-dtype fp8`. + +## 目的 + +当 vLLM 使用 **TP=2**（Tensor Parallel = 2）部署时，单张 H200 上可以同时运行 **4 个独立服务**（8 卡 / 2 = 4 服务），每个服务独占 2 张 GPU： + +| 服务 | 端口 | GPU 对 | +|------|------|--------| +| 1 | 30005 | 0, 1 | +| 2 | 30006 | 2, 3 | +| 3 | 30007 | 4, 5 | +| 4 | 30008 | 6, 7 | + +此时 `sglang.bench_serving` 存在以下问题： + +1. **低并发下无法测出真实吞吐**：并发数不大时，请求可能只落在一个服务上，其他 3 个服务空闲，bench 报告的吞吐只是单服务的 1/4。 +2. **TTFT 测量不够精确**：`sglang.bench_serving` 的计时粒度较粗，对于低延迟场景（TP=2 下 TTFT 可能 < 100ms）误差较大。 +3. **不支持多服务负载均衡**：只能测一个端口，无法反映整个 8 卡集群性能。 + +因此本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API，通过 `aiohttp` 并发发送请求，并记录每个请求的精确时间戳。同时通过 **负载均衡** 将请求均匀分发到 4 个服务上。 + +## 与 sglang.bench_serving 的区别 + +| 特性 | sglang.bench_serving | bench_client.py (本实验) | +|------|----------------------|--------------------------| +| 协议 | 内部协议 | OpenAI API (`/v1/chat/completions`) | +| 并发控制 | 进程级 | `asyncio.Semaphore` | +| TTFT 测量 | 粗略 | 精确到 `time.perf_counter()` | +| TPOT 测量 | 基于总时间估算 | 基于首 token 后每个 token 的间隔 | +| 多服务支持 | ❌ 单端口 | ✅ 轮询/随机分发到多个端口 | +| 低并发精度 | 一般 | 高（适合 TP=2 场景） | +| 集群吞吐 | 只能测单服务 | 可测 4 服务总吞吐 | + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡 | +| `config.env` | 实验配置（场景、模型路径、端口列表、LB 策略等） | +| `run_bench.sh` | 编排脚本：启动 4 个服务 → 运行压测 → 解析结果 | +| `start_server.sh` | 同时启动 4 个 vLLM TP=2 服务（每服务 2 GPU） | +| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | + +## 快速运行 + +```bash +# 完整流程（自动启动 4 个服务、压测、生成报告） +bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh + +# 跳过服务管理（已有 4 个服务在运行） +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh + +# 单独运行压测客户端（多服务，带健康检查） +python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ + --lb-strategy round_robin \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file /tmp/test.jsonl + +# 单独运行压测客户端（单服务，向后兼容） +python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ + --output-file /tmp/test_single.jsonl + +# 解析已有结果 +python3 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py \ + experiments/dsv4_h200_vllm_tp2_custom_bench/results/ +``` + +## 场景设计 + +本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： + +| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | +|------|----------|----------|--------|------| +| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | +| 4 | 512 | 256 | 80 | 低并发，4 服务各 1 req | +| 8 | 512 | 256 | 160 | 中低并发 | +| 16 | 512 | 256 | 320 | 中等并发 | +| 32 | 512 | 256 | 640 | 中高并发 | +| 64 | 512 | 256 | 1280 | 高并发 | +| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | +| 1 | 2048 | 512 | 20 | 长输入基线 | +| 8 | 2048 | 512 | 160 | 长输入中并发 | +| 32 | 2048 | 512 | 640 | 长输入高并发 | +| 128 | 2048 | 512 | 2560 | 长输入极限并发 | +| 1 | 4096 | 1024 | 20 | 超长输入基线 | +| 8 | 4096 | 1024 | 160 | 超长输入中并发 | +| 32 | 4096 | 1024 | 640 | 超长输入高并发 | +| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | + +## 健康检查与故障自动跳过 + +`bench_client.py` 内置了 **后台健康检查 + 请求级重试** 机制，确保某个服务挂了不会导致整个 benchmark 失败。 + +### 健康检查机制 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--health-check-interval` | 5.0 | 每 N 秒检查一次所有服务 | +| `--health-max-failures` | 2 | 连续失败 N 次才标记为不健康 | +| `--health-recovery-interval` | 10.0 | 不健康服务 N 秒后自动重试恢复 | +| `--request-max-retries` | 2 | 单个请求失败最多重试 N 次 | + +### 故障处理流程 + +``` +请求 → 负载均衡选择端口 → 发送请求 + ↓ +成功 → 记录结果 + ↓ +失败 → 换另一个端口重试（最多 --request-max-retries 次） + ↓ +全部失败 → 记录失败，继续下一个请求 +``` + +后台同时运行健康检查： +- 每 5 秒对所有端口执行 `/health` 探测 +- 连续 2 次失败的服务被标记为 **unhealthy**，不再接收新请求 +- 10 秒后自动尝试恢复，如果恢复成功则重新加入负载均衡池 + +### 输出中的故障信息 + +汇总 JSON 包含以下字段： + +```json +{ + "completed": 638, + "failed": 2, + "retried": 5, + "healthy_ports_at_end": [30005, 30006, 30007], + "per_port_stats": { + "30005": {"count": 160, "failed": 0, ...}, + "30006": {"count": 159, "failed": 1, ...}, + "30007": {"count": 160, "failed": 0, ...}, + "30008": {"count": 159, "failed": 1, ...} + } +} +``` + +- `failed`: 最终失败的请求数（重试后仍失败） +- `retried`: 至少重试过一次的请求数 +- `healthy_ports_at_end`: 压测结束时仍健康的服务端口 +- `per_port_stats[].failed`: 每个端口的失败数 + +### 场景间健康检查 + +`run_bench.sh` 在每个场景结束后会打印一次服务健康状态： + +``` +[2026-07-10T08:45:00+00:00] service health check: healthy=[30005 30006 30007 30008] unhealthy=[] +[2026-07-10T08:45:30+00:00] service health check: healthy=[30005 30006 30007] unhealthy=[30008] +[2026-07-10T08:45:30+00:00] WARNING: 1 service(s) unhealthy: 30008 +``` + +这样即使某个服务在压测中途 OOM 崩溃，其他服务仍能继续工作，benchmark 不会中断。 + +### 自定义故障参数 + +```bash +python3 bench_client.py \ + --ports 30005,30006,30007,30008 \ + --health-check-interval 3 \ + --health-max-failures 1 \ + --health-recovery-interval 5 \ + --request-max-retries 3 \ + ... +``` + +- 更敏感：间隔 3 秒、1 次失败即标记不健康、5 秒恢复 +- 更容错：单个请求最多重试 3 次 + +## 负载均衡策略 + +`bench_client.py` 支持三种负载均衡策略： + +| 策略 | 说明 | 适用场景 | +|------|------|----------| +| `round_robin` | 请求按顺序轮询分配到各端口 | **默认**，最公平，推荐 | +| `random` | 随机选择一个端口 | 简单，分布可能不均 | +| `least_conn` | 选择当前连接数最少的端口 | 需要追踪 in-flight，当前为 random 占位 | + +通过 `--lb-strategy` 参数选择： + +```bash +python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,30008 ... +``` + +## 输出格式 + +### JSONL 原始数据 (`raw_outputs/vllm_*.jsonl`) + +每行一个 JSON 对象，包含请求分发到的目标端口： + +```json +{ + "request_id": 0, + "input_tokens": 512, + "output_tokens": 256, + "first_token_time_ms": 45.23, + "e2e_latency_ms": 5234.56, + "inter_token_latencies": [18.5, 19.2, ...], + "tpot_ms": 19.8, + "mean_itl_ms": 19.5, + "success": true, + "error": "", + "target_port": 30005 +} +``` + +最后一行是汇总数据，包含 **各端口统计**： + +```json +{ + "completed": 640, + "failed": 0, + "duration": 12.34, + "request_throughput": 51.88, + "mean_ttft_ms": 48.5, + "p95_ttft_ms": 62.3, + "per_port_stats": { + "30005": {"count": 160, "mean_ttft_ms": 47.2, "p95_ttft_ms": 58.1, "mean_e2e_ms": 5200.0}, + "30006": {"count": 160, "mean_ttft_ms": 49.1, "p95_ttft_ms": 64.3, "mean_e2e_ms": 5250.0}, + "30007": {"count": 160, "mean_ttft_ms": 48.8, "p95_ttft_ms": 61.5, "mean_e2e_ms": 5230.0}, + "30008": {"count": 160, "mean_ttft_ms": 48.9, "p95_ttft_ms": 65.2, "mean_e2e_ms": 5270.0} + }, + "lb_strategy": "round_robin", + "num_services": 4 +} +``` + +### 结构化结果 (`results.json`) + +遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。 + +### 人类可读报告 (`report.md`) + +Markdown 表格，包含所有场景的 TTFT/TPOT/E2E 均值与分位数，以及 SLO 达标状态。 + +## 单独启动/停止 4 个服务 + +```bash +# 启动 4 个服务 +bash experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh + +# 停止（run_bench.sh 会自动停止，但也可以手动） +for port in 30005 30006 30007 30008; do + pid_file="experiments/dsv4_h200_vllm_tp2_custom_bench/server_port${port}.pid" + [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true + rm -f "$pid_file" +done +pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true +``` + +## 注意事项 + +- `bench_client.py` 优先使用 `aiohttp`，如果未安装则自动回退到 `urllib`（但会丢失 streaming 和精确 TTFT/ITL 测量）。 +- 默认 `temperature=0.0` 以保证输出长度稳定。 +- 输入 token 数通过空格分隔的单词数估算，实际 tokenizer 可能略有偏差。 +- 4 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 +- 每个服务占用约 2 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 +- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 +- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py new file mode 100755 index 0000000..d8edc36 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. + +Sends concurrent requests to multiple inference servers and records per-request +latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. + +Designed for TP=2 on 8-GPU setups where 4 independent services run on +(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via +round-robin or random load balancing. Unhealthy services are automatically +skipped with periodic retry. + +Usage: + # Single service (backward compatible) + python3 bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --output-file results.jsonl + + # Multi-service (4 services on ports 30005-30008) + python3 bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --lb-strategy round_robin \ + --output-file results.jsonl + +The output JSONL contains one object per request with raw timing data. +A final summary line ("completed" field) aggregates all requests. +""" + +import argparse +import asyncio +import json +import random +import sys +import time +import urllib.request +from dataclasses import asdict, dataclass, field +from typing import Any + +# Try aiohttp first; fall back to urllib for stdlib-only environments. +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + +@dataclass +class RequestResult: + request_id: int + input_tokens: int = 0 + output_tokens: int = 0 + first_token_time_ms: float = 0.0 # TTFT + e2e_latency_ms: float = 0.0 # total wall time + inter_token_latencies: list[float] = field(default_factory=list) + success: bool = False + error: str = "" + target_port: int = 0 # which service handled this request + retry_count: int = 0 # how many times this request was retried + + @property + def tpot_ms(self) -> float: + """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" + if self.output_tokens <= 1: + return 0.0 + return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) + + @property + def mean_itl_ms(self) -> float: + if not self.inter_token_latencies: + return 0.0 + return sum(self.inter_token_latencies) / len(self.inter_token_latencies) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["tpot_ms"] = self.tpot_ms + d["mean_itl_ms"] = self.mean_itl_ms + return d + + +# --------------------------------------------------------------------------- +# Health checker +# --------------------------------------------------------------------------- + +class HealthChecker: + """Periodically check service health and maintain a healthy-port list.""" + + def __init__( + self, + host: str, + ports: list[int], + check_interval: float = 5.0, + max_failures: int = 2, + recovery_interval: float = 10.0, + ): + self.host = host + self.all_ports = ports + self.check_interval = check_interval + self.max_failures = max_failures + self.recovery_interval = recovery_interval + + # port -> consecutive failure count + self._failures: dict[int, int] = {p: 0 for p in ports} + # port -> last healthy timestamp + self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} + self._lock = asyncio.Lock() + self._task: asyncio.Task | None = None + + async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: + url = f"http://{self.host}:{port}/health" + try: + if HAS_AIOHTTP and session is not None: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + return resp.status == 200 + else: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status == 200 + except Exception: + return False + + async def _check_all(self, session: aiohttp.ClientSession | None) -> None: + checks = [self._check_one(p, session) for p in self.all_ports] + results = await asyncio.gather(*checks, return_exceptions=True) + now = time.monotonic() + async with self._lock: + for port, ok in zip(self.all_ports, results): + if isinstance(ok, Exception): + ok = False + if ok: + self._failures[port] = 0 + self._last_healthy[port] = now + else: + self._failures[port] += 1 + + async def start(self, session: aiohttp.ClientSession | None) -> None: + async def _loop() -> None: + while True: + await asyncio.sleep(self.check_interval) + await self._check_all(session) + self._task = asyncio.create_task(_loop()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + @property + async def healthy_ports(self) -> list[int]: + now = time.monotonic() + async with self._lock: + healthy = [] + for p in self.all_ports: + if self._failures[p] < self.max_failures: + healthy.append(p) + elif now - self._last_healthy[p] > self.recovery_interval: + # Give it another chance after recovery_interval. + self._failures[p] = max(0, self._failures[p] - 1) + healthy.append(p) + return healthy + + async def is_healthy(self, port: int) -> bool: + ports = await self.healthy_ports + return port in ports + + +# --------------------------------------------------------------------------- +# Load balancer with failover +# --------------------------------------------------------------------------- + +class LoadBalancer: + """Distribute requests across healthy backend ports with auto-failover.""" + + def __init__( + self, + ports: list[int], + health_checker: HealthChecker, + strategy: str = "round_robin", + ): + self.all_ports = ports + self.health_checker = health_checker + self.strategy = strategy + self._idx = 0 + self._lock = asyncio.Lock() + + def _next_round_robin(self, healthy: list[int]) -> int: + if not healthy: + return self.all_ports[self._idx % len(self.all_ports)] + for _ in range(len(self.all_ports)): + port = self.all_ports[self._idx % len(self.all_ports)] + self._idx += 1 + if port in healthy: + return port + # Fallback: all unhealthy, pick first healthy anyway. + return healthy[0] if healthy else self.all_ports[0] + + def _next_random(self, healthy: list[int]) -> int: + if healthy: + return random.choice(healthy) + return random.choice(self.all_ports) + + async def next_port(self) -> int: + healthy = await self.health_checker.healthy_ports + async with self._lock: + if self.strategy == "round_robin": + return self._next_round_robin(healthy) + elif self.strategy in ("random", "least_conn"): + return self._next_random(healthy) + else: + return self._next_round_robin(healthy) + + @property + def port_count(self) -> int: + return len(self.all_ports) + + +# --------------------------------------------------------------------------- +# Token helpers +# --------------------------------------------------------------------------- + +def make_prompt(token_len: int, vocab_size: int = 32000) -> str: + """Generate a random-ish prompt of approximately `token_len` tokens.""" + words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", + "dog", "hello", "world", "deep", "seek", "model", "test", + "benchmark", "performance", "latency", "throughput", "token"] + needed = max(token_len, 1) + tokens: list[str] = [] + while len(tokens) < needed: + tokens.extend(words) + return " ".join(tokens[:needed]) + + +# --------------------------------------------------------------------------- +# aiohttp-based async request (preferred) +# --------------------------------------------------------------------------- + +async def send_request_aiohttp( + session: aiohttp.ClientSession, + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + headers = {"Content-Type": "application/json"} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + first_token_received = False + last_token_time = 0.0 + + try: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + text = await resp.text() + result.error = f"HTTP {resp.status}: {text[:200]}" + return result + + async for line in resp.content: + line = line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data_str = line[len("data: "):] + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + content = delta.get("content", "") + if content: + now = time.perf_counter() + if not first_token_received: + result.first_token_time_ms = (now - start_time) * 1000 + first_token_received = True + else: + result.inter_token_latencies.append((now - last_token_time) * 1000) + last_token_time = now + result.output_tokens += 1 + + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.input_tokens = len(prompt.split()) # approximate + result.success = True + + except asyncio.TimeoutError: + result.error = "timeout" + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# urllib-based synchronous request (fallback, no aiohttp) +# --------------------------------------------------------------------------- + +def send_request_urllib( + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + } + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=600) as resp: + body = json.loads(resp.read().decode("utf-8")) + + choices = body.get("choices", []) + if not choices: + result.error = "no choices in response" + return result + + content = choices[0].get("message", {}).get("content", "") + result.output_tokens = len(content.split()) if content else 0 + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.first_token_time_ms = result.e2e_latency_ms + result.input_tokens = len(prompt.split()) + result.success = True + + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# Unified send_request wrapper with retry & failover +# --------------------------------------------------------------------------- + +async def send_request_with_retry( + session: aiohttp.ClientSession | None, + host: str, + lb: LoadBalancer, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, + max_retries: int = 2, +) -> RequestResult: + """Send a request, retrying on failure with a different port each time.""" + result = RequestResult(request_id=request_id) + for attempt in range(max_retries + 1): + port = await lb.next_port() + if HAS_AIOHTTP and session is not None: + result = await send_request_aiohttp( + session, host, port, model, request_id, prompt, max_tokens, temperature + ) + else: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, send_request_urllib, + host, port, model, request_id, prompt, max_tokens, temperature + ) + result.retry_count = attempt + if result.success: + return result + # If failed, try another port on next iteration (unless last attempt). + if attempt < max_retries: + await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish + return result + + +# --------------------------------------------------------------------------- +# Benchmark orchestration +# --------------------------------------------------------------------------- + +async def run_benchmark( + host: str, + ports: list[int], + lb: LoadBalancer, + health_checker: HealthChecker, + model: str, + concurrency: int, + input_len: int, + output_len: int, + num_prompts: int, + temperature: float, + output_file: str, +) -> dict[str, Any]: + """Run the benchmark and write a JSONL file.""" + prompts = [make_prompt(input_len) for _ in range(num_prompts)] + results: list[RequestResult] = [] + semaphore = asyncio.Semaphore(concurrency) + + start = time.perf_counter() + + if HAS_AIOHTTP: + connector = aiohttp.TCPConnector(limit=concurrency * 2) + timeout = aiohttp.ClientTimeout(total=600, sock_read=300) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + await health_checker.start(session) + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + session, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + else: + await health_checker.start(None) + from concurrent.futures import ThreadPoolExecutor + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + None, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + + duration = time.perf_counter() - start + + # Write JSONL + with open(output_file, "w", encoding="utf-8") as f: + for r in results: + f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") + + # Compute summary + successful = [r for r in results if r.success] + failed = len(results) - len(successful) + retried = [r for r in results if r.retry_count > 0] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + ttfts = [r.first_token_time_ms for r in successful] + tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] + e2es = [r.e2e_latency_ms for r in successful] + itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] + + total_input_tokens = sum(r.input_tokens for r in successful) + total_output_tokens = sum(r.output_tokens for r in successful) + + # Per-port breakdown + per_port_stats: dict[int, dict[str, Any]] = {} + for port in ports: + port_results = [r for r in successful if r.target_port == port] + port_failed = [r for r in results if r.target_port == port and not r.success] + if port_results or port_failed: + port_ttfts = [r.first_token_time_ms for r in port_results] + port_e2es = [r.e2e_latency_ms for r in port_results] + per_port_stats[port] = { + "count": len(port_results), + "failed": len(port_failed), + "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, + "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, + "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, + } + + # Health-check stats + healthy_at_end = await health_checker.healthy_ports + + summary = { + "completed": len(successful), + "failed": failed, + "retried": len(retried), + "duration": duration, + "request_throughput": len(successful) / duration if duration > 0 else 0.0, + "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, + "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, + "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, + "median_ttft_ms": percentile(ttfts, 50), + "p90_ttft_ms": percentile(ttfts, 90), + "p95_ttft_ms": percentile(ttfts, 95), + "p99_ttft_ms": percentile(ttfts, 99), + "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, + "median_tpot_ms": percentile(tpots, 50), + "p90_tpot_ms": percentile(tpots, 90), + "p95_tpot_ms": percentile(tpots, 95), + "p99_tpot_ms": percentile(tpots, 99), + "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, + "median_e2e_latency_ms": percentile(e2es, 50), + "p90_e2e_latency_ms": percentile(e2es, 90), + "p95_e2e_latency_ms": percentile(e2es, 95), + "p99_e2e_latency_ms": percentile(e2es, 99), + "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, + "median_itl_ms": percentile(itls, 50), + "p90_itl_ms": percentile(itls, 90), + "p95_itl_ms": percentile(itls, 95), + "p99_itl_ms": percentile(itls, 99), + "per_port_stats": per_port_stats, + "healthy_ports_at_end": healthy_at_end, + "lb_strategy": lb.strategy, + "num_services": lb.port_count, + } + + # Append summary as the last line + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + + return summary + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30005, + help="Single service port (backward compatible)") + parser.add_argument("--ports", default="", + help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") + parser.add_argument("--model", default="deepseek-v4-flash") + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--input-len", type=int, default=512) + parser.add_argument("--output-len", type=int, default=256) + parser.add_argument("--num-prompts", type=int, default=10) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--lb-strategy", default="round_robin", + choices=["round_robin", "random", "least_conn"], + help="Load balancing strategy across services") + parser.add_argument("--health-check-interval", type=float, default=5.0, + help="Seconds between health checks (default: 5)") + parser.add_argument("--health-max-failures", type=int, default=2, + help="Consecutive failures before marking a port unhealthy (default: 2)") + parser.add_argument("--health-recovery-interval", type=float, default=10.0, + help="Seconds before retrying an unhealthy port (default: 10)") + parser.add_argument("--request-max-retries", type=int, default=2, + help="Max retries per request on failure (default: 2)") + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + + # Determine ports: --ports takes precedence, else fall back to --port. + if args.ports: + ports = [int(p.strip()) for p in args.ports.split(",")] + else: + ports = [args.port] + + health_checker = HealthChecker( + host=args.host, + ports=ports, + check_interval=args.health_check_interval, + max_failures=args.health_max_failures, + recovery_interval=args.health_recovery_interval, + ) + lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) + + summary = asyncio.run(run_benchmark( + host=args.host, + ports=ports, + lb=lb, + health_checker=health_checker, + model=args.model, + concurrency=args.concurrency, + input_len=args.input_len, + output_len=args.output_len, + num_prompts=args.num_prompts, + temperature=args.temperature, + output_file=args.output_file, + )) + + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env new file mode 100644 index 0000000..ddeb143 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env @@ -0,0 +1,84 @@ +# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench +# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. +# +# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: +# Service 1: port 30005, GPUs 0,1 +# Service 2: port 30006, GPUs 2,3 +# Service 3: port 30007, GPUs 4,5 +# Service 4: port 30008, GPUs 6,7 +# +# bench_client.py distributes requests across all 4 services via round-robin +# load balancing, giving accurate cluster-wide throughput even at low +# per-service concurrency. + +EXPERIMENT="dsv4_h200_vllm_tp2_custom_bench_no_fp8" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Primary port (backward compatible); multi-service ports are in PORTS. +PORT="30005" +PORTS="30005,30006,30007,30008" + +BACKEND="vllm" +ENGINE="vllm" + +# Native virtual environments on the host. +VENV_SERVER="/data/user1/yy/envs/vllm" +VENV_CLIENT="/data/user1/yy/envs/vllm" + +# Load balancing strategy: round_robin | random | least_conn +LB_STRATEGY="round_robin" + +# Benchmark scenarios: "concurrency input_len output_len num_prompts". +# For multi-service we include low-concurrency scenarios to show how +# requests spread across 4 services. +SCENARIOS=( + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + "1 2048 512 20" + "8 2048 512 160" + "32 2048 512 640" + "128 2048 512 2560" + "1 4096 1024 20" + "8 4096 1024 160" + "32 4096 1024 640" + "128 4096 1024 2560" + "1 8192 1024 20" + "8 8192 1024 160" + "32 8192 1024 640" + "128 8192 1024 2560" + "1 16384 1024 20" + "8 16384 1024 160" + "32 16384 1024 640" + "128 16384 1024 2560" + "1 32768 1024 20" + "8 32768 1024 160" + "32 32768 1024 640" + "128 32768 1024 2560" + "1 65536 1024 20" + "8 65536 1024 160" + "32 65536 1024 640" + "128 65536 1024 2560" + "1 131072 1024 20" + "8 131072 1024 160" + "32 131072 1024 640" + "128 131072 1024 2560" + "1 262144 1024 20" + "8 262144 1024 160" + "32 262144 1024 640" + "128 262144 1024 2560" + "1 524288 1024 20" + "8 524288 1024 160" + "32 524288 1024 640" + "128 524288 1024 2560" +) + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py new file mode 100755 index 0000000..eea1a2e --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. + +Reads JSONL files produced by bench_client.py (one request per line, +last line is the summary) and generates: + - results.json (appended scenarios, following the standard schema) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: + """Read the summary JSON (last line) and all per-request lines.""" + requests: list[dict] = [] + summary: dict | None = None + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The summary line has "completed" and "duration" keys. + if "completed" in obj and "duration" in obj: + summary = obj + else: + requests.append(obj) + return summary, requests + + +def compute_metrics(summary: dict, requests: list[dict]) -> dict: + completed = summary.get("completed", 0) + total = len(requests) + failed = total - completed if total > 0 else 0 + duration_s = summary.get("duration", 0.0) + + # Extract per-request metrics for percentiles. + ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] + tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] + e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] + itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": summary.get("request_throughput", 0.0), + "input_token_throughput": summary.get("input_throughput", 0.0), + "output_token_throughput": summary.get("output_throughput", 0.0), + "total_token_throughput": summary.get("total_throughput", 0.0), + "total_input_tokens": summary.get("total_input_tokens", 0), + "total_output_tokens": summary.get("total_output_tokens", 0), + "e2e_ms": { + "mean": summary.get("mean_e2e_latency_ms", 0.0), + "p50": percentile(e2es, 50), + "p90": percentile(e2es, 90), + "p95": percentile(e2es, 95), + "p99": percentile(e2es, 99), + }, + "ttft_ms": { + "mean": summary.get("mean_ttft_ms", 0.0), + "p50": percentile(ttfts, 50), + "p90": percentile(ttfts, 90), + "p95": percentile(ttfts, 95), + "p99": percentile(ttfts, 99), + }, + "tpot_ms": { + "mean": summary.get("mean_tpot_ms", 0.0), + "p50": percentile(tpots, 50), + "p90": percentile(tpots, 90), + "p95": percentile(tpots, 95), + "p99": percentile(tpots, 99), + }, + "itl_ms": { + "mean": summary.get("mean_itl_ms", 0.0), + "p50": percentile(itls, 50), + "p90": percentile(itls, 90), + "p95": percentile(itls, 95), + "p99": percentile(itls, 99), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") + f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") + f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") + f.write(f"- **Result root**: `{result_root}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + summary, requests = parse_jsonl(jsonl_path) + if summary is None: + continue + + metrics = compute_metrics(summary, requests) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json new file mode 100644 index 0000000..64dd782 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json @@ -0,0 +1,47 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp2_custom_bench", + "run_id": "20260710-091141", + "timestamp": "2026-07-10T09:11:41+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "39fb076", + "git_dirty": "dirty", + "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json new file mode 100644 index 0000000..3196c06 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json @@ -0,0 +1,47 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp2_custom_bench", + "run_id": "20260710-091522", + "timestamp": "2026-07-10T09:15:22+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "39fb076", + "git_dirty": "dirty", + "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json new file mode 100644 index 0000000..16c1892 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json @@ -0,0 +1,47 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp2_custom_bench", + "run_id": "20260710-091700", + "timestamp": "2026-07-10T09:17:00+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "39fb076", + "git_dirty": "dirty", + "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh new file mode 100755 index 0000000..6be99c8 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# Custom benchmark orchestrator for multi-service vLLM TP=2 on 8x H200. +# +# This runs 4 independent vLLM services (TP=2 each) on GPU pairs (0,1),(2,3),(4,5),(6,7). +# bench_client.py distributes requests across all 4 services via round-robin LB, +# giving accurate cluster-wide throughput even at low per-service concurrency. +# +# Usage: +# bash run_bench.sh +# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "services=${PORTS}" +log "lb_strategy=${LB_STRATEGY}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 4x vLLM TP=2 multi-service (no FP8 KV cache) benchmark using bench_client.py" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json, sys +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 2, + "num_services": 4, + "ports": "30005,30006,30007,30008", + "gpu_pairs": "0,1|2,3|4,5|6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", "4 512 256 80", "8 512 256 160", + "16 512 256 320", "32 512 256 640", "64 512 256 1280", "128 512 256 2560", + "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", + "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# --------------------------------------------------------------------------- +# Server helpers (multi-service) +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +are_all_services_healthy() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + for p in "${PORT_LIST[@]}"; do + if ! is_server_healthy "$p"; then + return 1 + fi + done + return 0 +} + +stop_all_servers() { + log "stopping all vllm services" + # Kill by PID files first. + for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do + [[ -f "$pid_file" ]] || continue + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "killing pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + done + # Fallback: kill any remaining vllm processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + sleep 2 +} + +start_all_servers() { + log "starting all vllm services with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + + if ! are_all_services_healthy "$PORTS"; then + log "error: not all vllm services became healthy" + exit 1 + fi + log "all vllm services are healthy" +} + +# --------------------------------------------------------------------------- +# Pre-benchmark health check with detailed reporting +# --------------------------------------------------------------------------- + +check_services_detailed() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy_count=0 + local unhealthy_ports="" + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + ((healthy_count++)) + else + unhealthy_ports="${unhealthy_ports}${p} " + fi + done + echo "$healthy_count" + if [[ -n "$unhealthy_ports" ]]; then + echo "unhealthy: ${unhealthy_ports}" >&2 + fi +} + +wait_for_all_services() { + local ports="$1" + local max_wait="${2:-240}" + local interval="${3:-5}" + local elapsed=0 + while (( elapsed < max_wait )); do + local healthy_count + healthy_count="$(check_services_detailed "$ports" | head -1)" + IFS=',' read -ra PORT_LIST <<< "$ports" + if (( healthy_count == ${#PORT_LIST[@]} )); then + return 0 + fi + log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" + sleep "$interval" + ((elapsed += interval)) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Mid-benchmark health check (called between scenarios) +# --------------------------------------------------------------------------- + +check_and_report_services() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy=() + local unhealthy=() + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + healthy+=("$p") + else + unhealthy+=("$p") + fi + done + log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" + if [[ ${#unhealthy[@]} -gt 0 ]]; then + log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" + fi + return ${#unhealthy[@]} +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_all_servers + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming services are already running" + if ! wait_for_all_services "$PORTS"; then + log "error: not all healthy services found on ports ${PORTS}" + exit 1 + fi +else + stop_all_servers + start_all_servers +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi + + output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" + + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ + --host 127.0.0.1 \ + --ports "$PORTS" \ + --model "$SERVED_MODEL_NAME" \ + --concurrency "$concurrency" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --lb-strategy "$LB_STRATEGY" \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file "$output_file" \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + # After failure, check service health before continuing. + check_and_report_services "$PORTS" + continue + } + + log "finished scenario: output=${output_file}" + + # Between scenarios, report service health so we can spot degradation early. + check_and_report_services "$PORTS" || true +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh new file mode 100755 index 0000000..8555596 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Start 4 independent vLLM TP=2 services on 8x H200. +# Each service runs on a pair of GPUs: (0,1), (2,3), (4,5), (6,7). +# This maximizes GPU utilization for low-concurrency workloads. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_SERVER}" +export PATH="$VENV/bin:$PATH" +VLLM="$VENV/bin/vllm" + +TP=2 +LOG_DIR="${SCRIPT_DIR}/logs" +mkdir -p "$LOG_DIR" + +# 4 services, each on 2 GPUs. +# Port and GPU pair mapping. +SERVICES=( + "30005:0,1" + "30006:2,3" + "30007:4,5" + "30008:6,7" +) + +# Kill any existing vLLM processes for this model. +pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true +sleep 2 + +# Remove old PID files. +rm -f "${SCRIPT_DIR}"/*.pid + +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + GPUS="${svc##*:}" + LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + + echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" + echo "Log: ${LOG}" + + CUDA_VISIBLE_DEVICES="${GPUS}" \ + nohup "$VLLM" serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --block-size 256 \ + --served-model-name "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + > "$LOG" 2>&1 & + PID=$! + echo $PID > "$PID_FILE" + echo "PID: $PID" +done + +echo "" +echo "Waiting for all 4 services to become healthy..." + +MAX_WAIT=240 +all_healthy=0 +for i in $(seq 1 $MAX_WAIT); do + all_healthy=1 + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + all_healthy=0 + break + fi + done + + if [[ "$all_healthy" -eq 1 ]]; then + echo "All 4 services are healthy!" + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + echo " - http://127.0.0.1:${PORT}" + done + exit 0 + fi + + # Check if any process died early. + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + if [[ -f "$PID_FILE" ]]; then + PID="$(cat "$PID_FILE")" + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + tail -200 $LOG 2>/dev/null || true + exit 1 + fi + fi + done + + if (( i % 10 == 0 )); then + echo "Waiting... (${i}/${MAX_WAIT})" + fi + sleep 5 +done + +echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + echo "--- Last 50 lines of port ${PORT} ---" + tail -50 $LOG 2>/dev/null || true +done +exit 1 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/README.md b/experiments/dsv4_h200_vllm_tp4_custom_bench/README.md new file mode 100644 index 0000000..1f724c6 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/README.md @@ -0,0 +1,124 @@ +# dsv4_h200_vllm_tp4_custom_bench + +## 目的 + +当 vLLM 使用 **TP=4**（Tensor Parallel = 4）部署时，单张 H200 上可以同时运行 **2 个独立服务**（8 卡 / 4 = 2 服务），每个服务独占 4 张 GPU： + +| 服务 | 端口 | GPU 组 | +|------|------|--------| +| 1 | 30005 | 0, 1, 2, 3 | +| 2 | 30006 | 4, 5, 6, 7 | + +相比 TP=2 方案，TP=4 的每个服务拥有更大的 tensor-parallel 宽度，适合更高并发的单服务场景。本实验同样使用 **自定义压测客户端 `bench_client.py`**，通过负载均衡将请求均匀分发到 2 个服务上。 + +## 与 TP=2 方案的区别 + +| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | +|------|---------------|---------------| +| 服务数量 | 4 | 2 | +| 每服务 GPU | 2 | 4 | +| 单服务吞吐 | 较低 | 较高 | +| 集群总吞吐 | 相近 | 相近 | +| 适用场景 | 低并发、低延迟 | 高并发、大 batch | + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡（复用 TP=2 版本） | +| `config.env` | 实验配置（2 服务、端口 30005/30006、场景等） | +| `run_bench.sh` | 编排脚本：启动 2 个服务 → 运行压测 → 解析结果 | +| `start_server.sh` | 同时启动 2 个 vLLM TP=4 服务（每服务 4 GPU） | +| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | + +## 快速运行 + +```bash +# 完整流程（自动启动 2 个服务、压测、生成报告） +bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh + +# 跳过服务管理（已有 2 个服务在运行） +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh + +# 单独运行压测客户端（多服务，带健康检查） +python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006 \ + --model deepseek-v4-flash \ + --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ + --lb-strategy round_robin \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file /tmp/test.jsonl + +# 单独运行压测客户端（单服务，向后兼容） +python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ + --output-file /tmp/test_single.jsonl + +# 解析已有结果 +python3 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py \ + experiments/dsv4_h200_vllm_tp4_custom_bench/results/ +``` + +## 场景设计 + +本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： + +| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | +|------|----------|----------|--------|------| +| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | +| 2 | 512 | 256 | 40 | 低并发，2 服务各 1 req | +| 4 | 512 | 256 | 80 | 中低并发 | +| 8 | 512 | 256 | 160 | 中等并发 | +| 16 | 512 | 256 | 320 | 中高并发 | +| 32 | 512 | 256 | 640 | 高并发 | +| 64 | 512 | 256 | 1280 | 很高并发 | +| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | +| 1 | 2048 | 512 | 20 | 长输入基线 | +| 8 | 2048 | 512 | 160 | 长输入中并发 | +| 32 | 2048 | 512 | 640 | 长输入高并发 | +| 128 | 2048 | 512 | 2560 | 长输入极限并发 | +| 1 | 4096 | 1024 | 20 | 超长输入基线 | +| 8 | 4096 | 1024 | 160 | 超长输入中并发 | +| 32 | 4096 | 1024 | 640 | 超长输入高并发 | +| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | + +## 健康检查与故障自动跳过 + +同 TP=2 版本，详见原 README。 + +## 负载均衡策略 + +同 TP=2 版本，支持 `round_robin`（默认）、`random`、`least_conn`。 + +## 输出格式 + +同 TP=2 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 + +## 单独启动/停止 2 个服务 + +```bash +# 启动 2 个服务 +bash experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh + +# 停止（run_bench.sh 会自动停止，但也可以手动） +for port in 30005 30006; do + pid_file="experiments/dsv4_h200_vllm_tp4_custom_bench/server_port${port}.pid" + [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true + rm -f "$pid_file" +done +pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true +``` + +## 注意事项 + +- 2 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 +- 每个服务占用约 4 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 +- TP=4 相比 TP=2 单服务延迟更低（通信更少），但服务数量减半，低并发下可能无法充分利用集群。 +- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 +- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py new file mode 100755 index 0000000..d8edc36 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. + +Sends concurrent requests to multiple inference servers and records per-request +latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. + +Designed for TP=2 on 8-GPU setups where 4 independent services run on +(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via +round-robin or random load balancing. Unhealthy services are automatically +skipped with periodic retry. + +Usage: + # Single service (backward compatible) + python3 bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --output-file results.jsonl + + # Multi-service (4 services on ports 30005-30008) + python3 bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --lb-strategy round_robin \ + --output-file results.jsonl + +The output JSONL contains one object per request with raw timing data. +A final summary line ("completed" field) aggregates all requests. +""" + +import argparse +import asyncio +import json +import random +import sys +import time +import urllib.request +from dataclasses import asdict, dataclass, field +from typing import Any + +# Try aiohttp first; fall back to urllib for stdlib-only environments. +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + +@dataclass +class RequestResult: + request_id: int + input_tokens: int = 0 + output_tokens: int = 0 + first_token_time_ms: float = 0.0 # TTFT + e2e_latency_ms: float = 0.0 # total wall time + inter_token_latencies: list[float] = field(default_factory=list) + success: bool = False + error: str = "" + target_port: int = 0 # which service handled this request + retry_count: int = 0 # how many times this request was retried + + @property + def tpot_ms(self) -> float: + """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" + if self.output_tokens <= 1: + return 0.0 + return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) + + @property + def mean_itl_ms(self) -> float: + if not self.inter_token_latencies: + return 0.0 + return sum(self.inter_token_latencies) / len(self.inter_token_latencies) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["tpot_ms"] = self.tpot_ms + d["mean_itl_ms"] = self.mean_itl_ms + return d + + +# --------------------------------------------------------------------------- +# Health checker +# --------------------------------------------------------------------------- + +class HealthChecker: + """Periodically check service health and maintain a healthy-port list.""" + + def __init__( + self, + host: str, + ports: list[int], + check_interval: float = 5.0, + max_failures: int = 2, + recovery_interval: float = 10.0, + ): + self.host = host + self.all_ports = ports + self.check_interval = check_interval + self.max_failures = max_failures + self.recovery_interval = recovery_interval + + # port -> consecutive failure count + self._failures: dict[int, int] = {p: 0 for p in ports} + # port -> last healthy timestamp + self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} + self._lock = asyncio.Lock() + self._task: asyncio.Task | None = None + + async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: + url = f"http://{self.host}:{port}/health" + try: + if HAS_AIOHTTP and session is not None: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + return resp.status == 200 + else: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status == 200 + except Exception: + return False + + async def _check_all(self, session: aiohttp.ClientSession | None) -> None: + checks = [self._check_one(p, session) for p in self.all_ports] + results = await asyncio.gather(*checks, return_exceptions=True) + now = time.monotonic() + async with self._lock: + for port, ok in zip(self.all_ports, results): + if isinstance(ok, Exception): + ok = False + if ok: + self._failures[port] = 0 + self._last_healthy[port] = now + else: + self._failures[port] += 1 + + async def start(self, session: aiohttp.ClientSession | None) -> None: + async def _loop() -> None: + while True: + await asyncio.sleep(self.check_interval) + await self._check_all(session) + self._task = asyncio.create_task(_loop()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + @property + async def healthy_ports(self) -> list[int]: + now = time.monotonic() + async with self._lock: + healthy = [] + for p in self.all_ports: + if self._failures[p] < self.max_failures: + healthy.append(p) + elif now - self._last_healthy[p] > self.recovery_interval: + # Give it another chance after recovery_interval. + self._failures[p] = max(0, self._failures[p] - 1) + healthy.append(p) + return healthy + + async def is_healthy(self, port: int) -> bool: + ports = await self.healthy_ports + return port in ports + + +# --------------------------------------------------------------------------- +# Load balancer with failover +# --------------------------------------------------------------------------- + +class LoadBalancer: + """Distribute requests across healthy backend ports with auto-failover.""" + + def __init__( + self, + ports: list[int], + health_checker: HealthChecker, + strategy: str = "round_robin", + ): + self.all_ports = ports + self.health_checker = health_checker + self.strategy = strategy + self._idx = 0 + self._lock = asyncio.Lock() + + def _next_round_robin(self, healthy: list[int]) -> int: + if not healthy: + return self.all_ports[self._idx % len(self.all_ports)] + for _ in range(len(self.all_ports)): + port = self.all_ports[self._idx % len(self.all_ports)] + self._idx += 1 + if port in healthy: + return port + # Fallback: all unhealthy, pick first healthy anyway. + return healthy[0] if healthy else self.all_ports[0] + + def _next_random(self, healthy: list[int]) -> int: + if healthy: + return random.choice(healthy) + return random.choice(self.all_ports) + + async def next_port(self) -> int: + healthy = await self.health_checker.healthy_ports + async with self._lock: + if self.strategy == "round_robin": + return self._next_round_robin(healthy) + elif self.strategy in ("random", "least_conn"): + return self._next_random(healthy) + else: + return self._next_round_robin(healthy) + + @property + def port_count(self) -> int: + return len(self.all_ports) + + +# --------------------------------------------------------------------------- +# Token helpers +# --------------------------------------------------------------------------- + +def make_prompt(token_len: int, vocab_size: int = 32000) -> str: + """Generate a random-ish prompt of approximately `token_len` tokens.""" + words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", + "dog", "hello", "world", "deep", "seek", "model", "test", + "benchmark", "performance", "latency", "throughput", "token"] + needed = max(token_len, 1) + tokens: list[str] = [] + while len(tokens) < needed: + tokens.extend(words) + return " ".join(tokens[:needed]) + + +# --------------------------------------------------------------------------- +# aiohttp-based async request (preferred) +# --------------------------------------------------------------------------- + +async def send_request_aiohttp( + session: aiohttp.ClientSession, + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + headers = {"Content-Type": "application/json"} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + first_token_received = False + last_token_time = 0.0 + + try: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + text = await resp.text() + result.error = f"HTTP {resp.status}: {text[:200]}" + return result + + async for line in resp.content: + line = line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data_str = line[len("data: "):] + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + content = delta.get("content", "") + if content: + now = time.perf_counter() + if not first_token_received: + result.first_token_time_ms = (now - start_time) * 1000 + first_token_received = True + else: + result.inter_token_latencies.append((now - last_token_time) * 1000) + last_token_time = now + result.output_tokens += 1 + + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.input_tokens = len(prompt.split()) # approximate + result.success = True + + except asyncio.TimeoutError: + result.error = "timeout" + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# urllib-based synchronous request (fallback, no aiohttp) +# --------------------------------------------------------------------------- + +def send_request_urllib( + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + } + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=600) as resp: + body = json.loads(resp.read().decode("utf-8")) + + choices = body.get("choices", []) + if not choices: + result.error = "no choices in response" + return result + + content = choices[0].get("message", {}).get("content", "") + result.output_tokens = len(content.split()) if content else 0 + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.first_token_time_ms = result.e2e_latency_ms + result.input_tokens = len(prompt.split()) + result.success = True + + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# Unified send_request wrapper with retry & failover +# --------------------------------------------------------------------------- + +async def send_request_with_retry( + session: aiohttp.ClientSession | None, + host: str, + lb: LoadBalancer, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, + max_retries: int = 2, +) -> RequestResult: + """Send a request, retrying on failure with a different port each time.""" + result = RequestResult(request_id=request_id) + for attempt in range(max_retries + 1): + port = await lb.next_port() + if HAS_AIOHTTP and session is not None: + result = await send_request_aiohttp( + session, host, port, model, request_id, prompt, max_tokens, temperature + ) + else: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, send_request_urllib, + host, port, model, request_id, prompt, max_tokens, temperature + ) + result.retry_count = attempt + if result.success: + return result + # If failed, try another port on next iteration (unless last attempt). + if attempt < max_retries: + await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish + return result + + +# --------------------------------------------------------------------------- +# Benchmark orchestration +# --------------------------------------------------------------------------- + +async def run_benchmark( + host: str, + ports: list[int], + lb: LoadBalancer, + health_checker: HealthChecker, + model: str, + concurrency: int, + input_len: int, + output_len: int, + num_prompts: int, + temperature: float, + output_file: str, +) -> dict[str, Any]: + """Run the benchmark and write a JSONL file.""" + prompts = [make_prompt(input_len) for _ in range(num_prompts)] + results: list[RequestResult] = [] + semaphore = asyncio.Semaphore(concurrency) + + start = time.perf_counter() + + if HAS_AIOHTTP: + connector = aiohttp.TCPConnector(limit=concurrency * 2) + timeout = aiohttp.ClientTimeout(total=600, sock_read=300) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + await health_checker.start(session) + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + session, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + else: + await health_checker.start(None) + from concurrent.futures import ThreadPoolExecutor + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + None, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + + duration = time.perf_counter() - start + + # Write JSONL + with open(output_file, "w", encoding="utf-8") as f: + for r in results: + f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") + + # Compute summary + successful = [r for r in results if r.success] + failed = len(results) - len(successful) + retried = [r for r in results if r.retry_count > 0] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + ttfts = [r.first_token_time_ms for r in successful] + tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] + e2es = [r.e2e_latency_ms for r in successful] + itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] + + total_input_tokens = sum(r.input_tokens for r in successful) + total_output_tokens = sum(r.output_tokens for r in successful) + + # Per-port breakdown + per_port_stats: dict[int, dict[str, Any]] = {} + for port in ports: + port_results = [r for r in successful if r.target_port == port] + port_failed = [r for r in results if r.target_port == port and not r.success] + if port_results or port_failed: + port_ttfts = [r.first_token_time_ms for r in port_results] + port_e2es = [r.e2e_latency_ms for r in port_results] + per_port_stats[port] = { + "count": len(port_results), + "failed": len(port_failed), + "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, + "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, + "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, + } + + # Health-check stats + healthy_at_end = await health_checker.healthy_ports + + summary = { + "completed": len(successful), + "failed": failed, + "retried": len(retried), + "duration": duration, + "request_throughput": len(successful) / duration if duration > 0 else 0.0, + "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, + "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, + "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, + "median_ttft_ms": percentile(ttfts, 50), + "p90_ttft_ms": percentile(ttfts, 90), + "p95_ttft_ms": percentile(ttfts, 95), + "p99_ttft_ms": percentile(ttfts, 99), + "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, + "median_tpot_ms": percentile(tpots, 50), + "p90_tpot_ms": percentile(tpots, 90), + "p95_tpot_ms": percentile(tpots, 95), + "p99_tpot_ms": percentile(tpots, 99), + "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, + "median_e2e_latency_ms": percentile(e2es, 50), + "p90_e2e_latency_ms": percentile(e2es, 90), + "p95_e2e_latency_ms": percentile(e2es, 95), + "p99_e2e_latency_ms": percentile(e2es, 99), + "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, + "median_itl_ms": percentile(itls, 50), + "p90_itl_ms": percentile(itls, 90), + "p95_itl_ms": percentile(itls, 95), + "p99_itl_ms": percentile(itls, 99), + "per_port_stats": per_port_stats, + "healthy_ports_at_end": healthy_at_end, + "lb_strategy": lb.strategy, + "num_services": lb.port_count, + } + + # Append summary as the last line + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + + return summary + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30005, + help="Single service port (backward compatible)") + parser.add_argument("--ports", default="", + help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") + parser.add_argument("--model", default="deepseek-v4-flash") + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--input-len", type=int, default=512) + parser.add_argument("--output-len", type=int, default=256) + parser.add_argument("--num-prompts", type=int, default=10) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--lb-strategy", default="round_robin", + choices=["round_robin", "random", "least_conn"], + help="Load balancing strategy across services") + parser.add_argument("--health-check-interval", type=float, default=5.0, + help="Seconds between health checks (default: 5)") + parser.add_argument("--health-max-failures", type=int, default=2, + help="Consecutive failures before marking a port unhealthy (default: 2)") + parser.add_argument("--health-recovery-interval", type=float, default=10.0, + help="Seconds before retrying an unhealthy port (default: 10)") + parser.add_argument("--request-max-retries", type=int, default=2, + help="Max retries per request on failure (default: 2)") + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + + # Determine ports: --ports takes precedence, else fall back to --port. + if args.ports: + ports = [int(p.strip()) for p in args.ports.split(",")] + else: + ports = [args.port] + + health_checker = HealthChecker( + host=args.host, + ports=ports, + check_interval=args.health_check_interval, + max_failures=args.health_max_failures, + recovery_interval=args.health_recovery_interval, + ) + lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) + + summary = asyncio.run(run_benchmark( + host=args.host, + ports=ports, + lb=lb, + health_checker=health_checker, + model=args.model, + concurrency=args.concurrency, + input_len=args.input_len, + output_len=args.output_len, + num_prompts=args.num_prompts, + temperature=args.temperature, + output_file=args.output_file, + )) + + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env new file mode 100644 index 0000000..365ef84 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env @@ -0,0 +1,84 @@ +# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench +# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. +# +# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: +# Service 1: port 30005, GPUs 0,1 +# Service 2: port 30006, GPUs 2,3 +# Service 3: port 30007, GPUs 4,5 +# Service 4: port 30008, GPUs 6,7 +# +# bench_client.py distributes requests across all 4 services via round-robin +# load balancing, giving accurate cluster-wide throughput even at low +# per-service concurrency. + +EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Primary port (backward compatible); multi-service ports are in PORTS. +PORT="30005" +PORTS="30005,30006" + +BACKEND="vllm" +ENGINE="vllm" + +# Native virtual environments on the host. +VENV_SERVER="/data/user1/yy/envs/vllm" +VENV_CLIENT="/data/user1/yy/envs/vllm" + +# Load balancing strategy: round_robin | random | least_conn +LB_STRATEGY="round_robin" + +# Benchmark scenarios: "concurrency input_len output_len num_prompts". +# For multi-service we include low-concurrency scenarios to show how +# requests spread across 2 services. +SCENARIOS=( + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + "1 2048 512 20" + "8 2048 512 160" + "32 2048 512 640" + "128 2048 512 2560" + "1 4096 1024 20" + "8 4096 1024 160" + "32 4096 1024 640" + "128 4096 1024 2560" + "1 8192 1024 20" + "8 8192 1024 160" + "32 8192 1024 640" + "128 8192 1024 2560" + "1 16384 1024 20" + "8 16384 1024 160" + "32 16384 1024 640" + "128 16384 1024 2560" + "1 32768 1024 20" + "8 32768 1024 160" + "32 32768 1024 640" + "128 32768 1024 2560" + "1 65536 1024 20" + "8 65536 1024 160" + "32 65536 1024 640" + "128 65536 1024 2560" + "1 131072 1024 20" + "8 131072 1024 160" + "32 131072 1024 640" + "128 131072 1024 2560" + "1 262144 1024 20" + "8 262144 1024 160" + "32 262144 1024 640" + "128 262144 1024 2560" + "1 524288 1024 20" + "8 524288 1024 160" + "32 524288 1024 640" + "128 524288 1024 2560" +) + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py b/experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py new file mode 100755 index 0000000..eea1a2e --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. + +Reads JSONL files produced by bench_client.py (one request per line, +last line is the summary) and generates: + - results.json (appended scenarios, following the standard schema) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: + """Read the summary JSON (last line) and all per-request lines.""" + requests: list[dict] = [] + summary: dict | None = None + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The summary line has "completed" and "duration" keys. + if "completed" in obj and "duration" in obj: + summary = obj + else: + requests.append(obj) + return summary, requests + + +def compute_metrics(summary: dict, requests: list[dict]) -> dict: + completed = summary.get("completed", 0) + total = len(requests) + failed = total - completed if total > 0 else 0 + duration_s = summary.get("duration", 0.0) + + # Extract per-request metrics for percentiles. + ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] + tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] + e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] + itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": summary.get("request_throughput", 0.0), + "input_token_throughput": summary.get("input_throughput", 0.0), + "output_token_throughput": summary.get("output_throughput", 0.0), + "total_token_throughput": summary.get("total_throughput", 0.0), + "total_input_tokens": summary.get("total_input_tokens", 0), + "total_output_tokens": summary.get("total_output_tokens", 0), + "e2e_ms": { + "mean": summary.get("mean_e2e_latency_ms", 0.0), + "p50": percentile(e2es, 50), + "p90": percentile(e2es, 90), + "p95": percentile(e2es, 95), + "p99": percentile(e2es, 99), + }, + "ttft_ms": { + "mean": summary.get("mean_ttft_ms", 0.0), + "p50": percentile(ttfts, 50), + "p90": percentile(ttfts, 90), + "p95": percentile(ttfts, 95), + "p99": percentile(ttfts, 99), + }, + "tpot_ms": { + "mean": summary.get("mean_tpot_ms", 0.0), + "p50": percentile(tpots, 50), + "p90": percentile(tpots, 90), + "p95": percentile(tpots, 95), + "p99": percentile(tpots, 99), + }, + "itl_ms": { + "mean": summary.get("mean_itl_ms", 0.0), + "p50": percentile(itls, 50), + "p90": percentile(itls, 90), + "p95": percentile(itls, 95), + "p99": percentile(itls, 99), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") + f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") + f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") + f.write(f"- **Result root**: `{result_root}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + summary, requests = parse_jsonl(jsonl_path) + if summary is None: + continue + + metrics = compute_metrics(summary, requests) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh new file mode 100755 index 0000000..ccf7eee --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +# Custom benchmark orchestrator for multi-service vLLM TP=4 on 8x H200. +# +# This runs 2 independent vLLM services (TP=4 each) on GPU groups (0,1,2,3) and (4,5,6,7). +# bench_client.py distributes requests across all 2 services via round-robin LB, +# giving accurate cluster-wide throughput even at low per-service concurrency. +# +# Usage: +# bash run_bench.sh +# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "services=${PORTS}" +log "lb_strategy=${LB_STRATEGY}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 2x vLLM TP=4 multi-service benchmark using bench_client.py" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json, sys +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 4, + "num_services": 2, + "ports": "30005,30006", + "gpu_groups": "0,1,2,3|4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", "2 512 256 40", "4 512 256 80", + "8 512 256 160", "16 512 256 320", "32 512 256 640", + "64 512 256 1280", "128 512 256 2560", + "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", + "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# --------------------------------------------------------------------------- +# Server helpers (multi-service) +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +are_all_services_healthy() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + for p in "${PORT_LIST[@]}"; do + if ! is_server_healthy "$p"; then + return 1 + fi + done + return 0 +} + +stop_all_servers() { + log "stopping all vllm services" + # Kill by PID files first. + for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do + [[ -f "$pid_file" ]] || continue + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "killing pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + done + # Fallback: kill any remaining vllm processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + sleep 2 +} + +start_all_servers() { + log "starting all vllm services with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + + if ! are_all_services_healthy "$PORTS"; then + log "error: not all vllm services became healthy" + exit 1 + fi + log "all vllm services are healthy" +} + +# --------------------------------------------------------------------------- +# Pre-benchmark health check with detailed reporting +# --------------------------------------------------------------------------- + +check_services_detailed() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy_count=0 + local unhealthy_ports="" + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + ((healthy_count++)) + else + unhealthy_ports="${unhealthy_ports}${p} " + fi + done + echo "$healthy_count" + if [[ -n "$unhealthy_ports" ]]; then + echo "unhealthy: ${unhealthy_ports}" >&2 + fi +} + +wait_for_all_services() { + local ports="$1" + local max_wait="${2:-240}" + local interval="${3:-5}" + local elapsed=0 + while (( elapsed < max_wait )); do + local healthy_count + healthy_count="$(check_services_detailed "$ports" | head -1)" + IFS=',' read -ra PORT_LIST <<< "$ports" + if (( healthy_count == ${#PORT_LIST[@]} )); then + return 0 + fi + log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" + sleep "$interval" + ((elapsed += interval)) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Mid-benchmark health check (called between scenarios) +# --------------------------------------------------------------------------- + +check_and_report_services() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy=() + local unhealthy=() + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + healthy+=("$p") + else + unhealthy+=("$p") + fi + done + log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" + if [[ ${#unhealthy[@]} -gt 0 ]]; then + log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" + fi + return ${#unhealthy[@]} +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_all_servers + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming services are already running" + if ! wait_for_all_services "$PORTS"; then + log "error: not all healthy services found on ports ${PORTS}" + exit 1 + fi +else + stop_all_servers + start_all_servers +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi + + output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" + + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ + --host 127.0.0.1 \ + --ports "$PORTS" \ + --model "$SERVED_MODEL_NAME" \ + --concurrency "$concurrency" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --lb-strategy "$LB_STRATEGY" \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file "$output_file" \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + # After failure, check service health before continuing. + check_and_report_services "$PORTS" + continue + } + + log "finished scenario: output=${output_file}" + + # Between scenarios, report service health so we can spot degradation early. + check_and_report_services "$PORTS" || true +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh new file mode 100755 index 0000000..ea279a9 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Start 2 independent vLLM TP=4 services on 8x H200. +# Each service runs on 4 GPUs: (0,1,2,3) and (4,5,6,7). +# This maximizes per-service throughput for higher-concurrency workloads. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_SERVER}" +export PATH="$VENV/bin:$PATH" +VLLM="$VENV/bin/vllm" + +TP=4 +LOG_DIR="${SCRIPT_DIR}/logs" +mkdir -p "$LOG_DIR" + +# 2 services, each on 4 GPUs. +# Port and GPU mapping. +SERVICES=( + "30005:0,1,2,3" + "30006:4,5,6,7" +) + +# Kill any existing vLLM processes for this model. +pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true +sleep 2 + +# Remove old PID files. +rm -f "${SCRIPT_DIR}"/*.pid + +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + GPUS="${svc##*:}" + LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + + echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" + echo "Log: ${LOG}" + + CUDA_VISIBLE_DEVICES="${GPUS}" \ + nohup "$VLLM" serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --block-size 256 \ + --served-model-name "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + > "$LOG" 2>&1 & + PID=$! + echo $PID > "$PID_FILE" + echo "PID: $PID" +done + +echo "" +echo "Waiting for all 2 services to become healthy..." + +MAX_WAIT=240 +all_healthy=0 +for i in $(seq 1 $MAX_WAIT); do + all_healthy=1 + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + all_healthy=0 + break + fi + done + + if [[ "$all_healthy" -eq 1 ]]; then + echo "All 2 services are healthy!" + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + echo " - http://127.0.0.1:${PORT}" + done + exit 0 + fi + + # Check if any process died early. + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + if [[ -f "$PID_FILE" ]]; then + PID="$(cat "$PID_FILE")" + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + tail -200 $LOG 2>/dev/null || true + exit 1 + fi + fi + done + + if (( i % 10 == 0 )); then + echo "Waiting... (${i}/${MAX_WAIT})" + fi + sleep 5 +done + +echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + echo "--- Last 50 lines of port ${PORT} ---" + tail -50 $LOG 2>/dev/null || true +done +exit 1 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md new file mode 100644 index 0000000..288b634 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md @@ -0,0 +1,126 @@ +# dsv4_h200_vllm_tp4_custom_bench_no_fp8 + +## 目的 + +当 vLLM 使用 **TP=4**（Tensor Parallel = 4）部署时，单张 H200 上可以同时运行 **2 个独立服务**（8 卡 / 4 = 2 服务），每个服务独占 4 张 GPU： + +> **注意**：本实验变体 **不启用 `--kv-cache-dtype fp8`**，用于对比 FP8 KV Cache 与默认精度的性能差异。 + +| 服务 | 端口 | GPU 组 | +|------|------|--------| +| 1 | 30005 | 0, 1, 2, 3 | +| 2 | 30006 | 4, 5, 6, 7 | + +相比 TP=2 方案，TP=4 的每个服务拥有更大的 tensor-parallel 宽度，适合更高并发的单服务场景。本实验同样使用 **自定义压测客户端 `bench_client.py`**，通过负载均衡将请求均匀分发到 2 个服务上。 + +## 与 TP=2 方案的区别 + +| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | +|------|---------------|---------------| +| 服务数量 | 4 | 2 | +| 每服务 GPU | 2 | 4 | +| 单服务吞吐 | 较低 | 较高 | +| 集群总吞吐 | 相近 | 相近 | +| 适用场景 | 低并发、低延迟 | 高并发、大 batch | + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡（复用 TP=2 版本） | +| `config.env` | 实验配置（2 服务、端口 30005/30006、场景等） | +| `run_bench.sh` | 编排脚本：启动 2 个服务 → 运行压测 → 解析结果 | +| `start_server.sh` | 同时启动 2 个 vLLM TP=4 服务（每服务 4 GPU） | +| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | + +## 快速运行 + +```bash +# 完整流程（自动启动 2 个服务、压测、生成报告） +bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh + +# 跳过服务管理（已有 2 个服务在运行） +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh + +# 单独运行压测客户端（多服务，带健康检查） +python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006 \ + --model deepseek-v4-flash \ + --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ + --lb-strategy round_robin \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file /tmp/test.jsonl + +# 单独运行压测客户端（单服务，向后兼容） +python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ + --output-file /tmp/test_single.jsonl + +# 解析已有结果 +python3 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py \ + experiments/dsv4_h200_vllm_tp4_custom_bench/results/ +``` + +## 场景设计 + +本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： + +| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | +|------|----------|----------|--------|------| +| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | +| 2 | 512 | 256 | 40 | 低并发，2 服务各 1 req | +| 4 | 512 | 256 | 80 | 中低并发 | +| 8 | 512 | 256 | 160 | 中等并发 | +| 16 | 512 | 256 | 320 | 中高并发 | +| 32 | 512 | 256 | 640 | 高并发 | +| 64 | 512 | 256 | 1280 | 很高并发 | +| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | +| 1 | 2048 | 512 | 20 | 长输入基线 | +| 8 | 2048 | 512 | 160 | 长输入中并发 | +| 32 | 2048 | 512 | 640 | 长输入高并发 | +| 128 | 2048 | 512 | 2560 | 长输入极限并发 | +| 1 | 4096 | 1024 | 20 | 超长输入基线 | +| 8 | 4096 | 1024 | 160 | 超长输入中并发 | +| 32 | 4096 | 1024 | 640 | 超长输入高并发 | +| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | + +## 健康检查与故障自动跳过 + +同 TP=2 版本，详见原 README。 + +## 负载均衡策略 + +同 TP=2 版本，支持 `round_robin`（默认）、`random`、`least_conn`。 + +## 输出格式 + +同 TP=2 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 + +## 单独启动/停止 2 个服务 + +```bash +# 启动 2 个服务 +bash experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh + +# 停止（run_bench.sh 会自动停止，但也可以手动） +for port in 30005 30006; do + pid_file="experiments/dsv4_h200_vllm_tp4_custom_bench/server_port${port}.pid" + [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true + rm -f "$pid_file" +done +pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true +``` + +## 注意事项 + +- 2 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 +- 每个服务占用约 4 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 +- TP=4 相比 TP=2 单服务延迟更低（通信更少），但服务数量减半，低并发下可能无法充分利用集群。 +- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 +- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py new file mode 100755 index 0000000..d8edc36 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. + +Sends concurrent requests to multiple inference servers and records per-request +latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. + +Designed for TP=2 on 8-GPU setups where 4 independent services run on +(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via +round-robin or random load balancing. Unhealthy services are automatically +skipped with periodic retry. + +Usage: + # Single service (backward compatible) + python3 bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --output-file results.jsonl + + # Multi-service (4 services on ports 30005-30008) + python3 bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --lb-strategy round_robin \ + --output-file results.jsonl + +The output JSONL contains one object per request with raw timing data. +A final summary line ("completed" field) aggregates all requests. +""" + +import argparse +import asyncio +import json +import random +import sys +import time +import urllib.request +from dataclasses import asdict, dataclass, field +from typing import Any + +# Try aiohttp first; fall back to urllib for stdlib-only environments. +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + +@dataclass +class RequestResult: + request_id: int + input_tokens: int = 0 + output_tokens: int = 0 + first_token_time_ms: float = 0.0 # TTFT + e2e_latency_ms: float = 0.0 # total wall time + inter_token_latencies: list[float] = field(default_factory=list) + success: bool = False + error: str = "" + target_port: int = 0 # which service handled this request + retry_count: int = 0 # how many times this request was retried + + @property + def tpot_ms(self) -> float: + """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" + if self.output_tokens <= 1: + return 0.0 + return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) + + @property + def mean_itl_ms(self) -> float: + if not self.inter_token_latencies: + return 0.0 + return sum(self.inter_token_latencies) / len(self.inter_token_latencies) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["tpot_ms"] = self.tpot_ms + d["mean_itl_ms"] = self.mean_itl_ms + return d + + +# --------------------------------------------------------------------------- +# Health checker +# --------------------------------------------------------------------------- + +class HealthChecker: + """Periodically check service health and maintain a healthy-port list.""" + + def __init__( + self, + host: str, + ports: list[int], + check_interval: float = 5.0, + max_failures: int = 2, + recovery_interval: float = 10.0, + ): + self.host = host + self.all_ports = ports + self.check_interval = check_interval + self.max_failures = max_failures + self.recovery_interval = recovery_interval + + # port -> consecutive failure count + self._failures: dict[int, int] = {p: 0 for p in ports} + # port -> last healthy timestamp + self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} + self._lock = asyncio.Lock() + self._task: asyncio.Task | None = None + + async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: + url = f"http://{self.host}:{port}/health" + try: + if HAS_AIOHTTP and session is not None: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + return resp.status == 200 + else: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status == 200 + except Exception: + return False + + async def _check_all(self, session: aiohttp.ClientSession | None) -> None: + checks = [self._check_one(p, session) for p in self.all_ports] + results = await asyncio.gather(*checks, return_exceptions=True) + now = time.monotonic() + async with self._lock: + for port, ok in zip(self.all_ports, results): + if isinstance(ok, Exception): + ok = False + if ok: + self._failures[port] = 0 + self._last_healthy[port] = now + else: + self._failures[port] += 1 + + async def start(self, session: aiohttp.ClientSession | None) -> None: + async def _loop() -> None: + while True: + await asyncio.sleep(self.check_interval) + await self._check_all(session) + self._task = asyncio.create_task(_loop()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + @property + async def healthy_ports(self) -> list[int]: + now = time.monotonic() + async with self._lock: + healthy = [] + for p in self.all_ports: + if self._failures[p] < self.max_failures: + healthy.append(p) + elif now - self._last_healthy[p] > self.recovery_interval: + # Give it another chance after recovery_interval. + self._failures[p] = max(0, self._failures[p] - 1) + healthy.append(p) + return healthy + + async def is_healthy(self, port: int) -> bool: + ports = await self.healthy_ports + return port in ports + + +# --------------------------------------------------------------------------- +# Load balancer with failover +# --------------------------------------------------------------------------- + +class LoadBalancer: + """Distribute requests across healthy backend ports with auto-failover.""" + + def __init__( + self, + ports: list[int], + health_checker: HealthChecker, + strategy: str = "round_robin", + ): + self.all_ports = ports + self.health_checker = health_checker + self.strategy = strategy + self._idx = 0 + self._lock = asyncio.Lock() + + def _next_round_robin(self, healthy: list[int]) -> int: + if not healthy: + return self.all_ports[self._idx % len(self.all_ports)] + for _ in range(len(self.all_ports)): + port = self.all_ports[self._idx % len(self.all_ports)] + self._idx += 1 + if port in healthy: + return port + # Fallback: all unhealthy, pick first healthy anyway. + return healthy[0] if healthy else self.all_ports[0] + + def _next_random(self, healthy: list[int]) -> int: + if healthy: + return random.choice(healthy) + return random.choice(self.all_ports) + + async def next_port(self) -> int: + healthy = await self.health_checker.healthy_ports + async with self._lock: + if self.strategy == "round_robin": + return self._next_round_robin(healthy) + elif self.strategy in ("random", "least_conn"): + return self._next_random(healthy) + else: + return self._next_round_robin(healthy) + + @property + def port_count(self) -> int: + return len(self.all_ports) + + +# --------------------------------------------------------------------------- +# Token helpers +# --------------------------------------------------------------------------- + +def make_prompt(token_len: int, vocab_size: int = 32000) -> str: + """Generate a random-ish prompt of approximately `token_len` tokens.""" + words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", + "dog", "hello", "world", "deep", "seek", "model", "test", + "benchmark", "performance", "latency", "throughput", "token"] + needed = max(token_len, 1) + tokens: list[str] = [] + while len(tokens) < needed: + tokens.extend(words) + return " ".join(tokens[:needed]) + + +# --------------------------------------------------------------------------- +# aiohttp-based async request (preferred) +# --------------------------------------------------------------------------- + +async def send_request_aiohttp( + session: aiohttp.ClientSession, + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + headers = {"Content-Type": "application/json"} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + first_token_received = False + last_token_time = 0.0 + + try: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + text = await resp.text() + result.error = f"HTTP {resp.status}: {text[:200]}" + return result + + async for line in resp.content: + line = line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data_str = line[len("data: "):] + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + content = delta.get("content", "") + if content: + now = time.perf_counter() + if not first_token_received: + result.first_token_time_ms = (now - start_time) * 1000 + first_token_received = True + else: + result.inter_token_latencies.append((now - last_token_time) * 1000) + last_token_time = now + result.output_tokens += 1 + + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.input_tokens = len(prompt.split()) # approximate + result.success = True + + except asyncio.TimeoutError: + result.error = "timeout" + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# urllib-based synchronous request (fallback, no aiohttp) +# --------------------------------------------------------------------------- + +def send_request_urllib( + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + } + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=600) as resp: + body = json.loads(resp.read().decode("utf-8")) + + choices = body.get("choices", []) + if not choices: + result.error = "no choices in response" + return result + + content = choices[0].get("message", {}).get("content", "") + result.output_tokens = len(content.split()) if content else 0 + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.first_token_time_ms = result.e2e_latency_ms + result.input_tokens = len(prompt.split()) + result.success = True + + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# Unified send_request wrapper with retry & failover +# --------------------------------------------------------------------------- + +async def send_request_with_retry( + session: aiohttp.ClientSession | None, + host: str, + lb: LoadBalancer, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, + max_retries: int = 2, +) -> RequestResult: + """Send a request, retrying on failure with a different port each time.""" + result = RequestResult(request_id=request_id) + for attempt in range(max_retries + 1): + port = await lb.next_port() + if HAS_AIOHTTP and session is not None: + result = await send_request_aiohttp( + session, host, port, model, request_id, prompt, max_tokens, temperature + ) + else: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, send_request_urllib, + host, port, model, request_id, prompt, max_tokens, temperature + ) + result.retry_count = attempt + if result.success: + return result + # If failed, try another port on next iteration (unless last attempt). + if attempt < max_retries: + await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish + return result + + +# --------------------------------------------------------------------------- +# Benchmark orchestration +# --------------------------------------------------------------------------- + +async def run_benchmark( + host: str, + ports: list[int], + lb: LoadBalancer, + health_checker: HealthChecker, + model: str, + concurrency: int, + input_len: int, + output_len: int, + num_prompts: int, + temperature: float, + output_file: str, +) -> dict[str, Any]: + """Run the benchmark and write a JSONL file.""" + prompts = [make_prompt(input_len) for _ in range(num_prompts)] + results: list[RequestResult] = [] + semaphore = asyncio.Semaphore(concurrency) + + start = time.perf_counter() + + if HAS_AIOHTTP: + connector = aiohttp.TCPConnector(limit=concurrency * 2) + timeout = aiohttp.ClientTimeout(total=600, sock_read=300) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + await health_checker.start(session) + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + session, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + else: + await health_checker.start(None) + from concurrent.futures import ThreadPoolExecutor + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + None, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + + duration = time.perf_counter() - start + + # Write JSONL + with open(output_file, "w", encoding="utf-8") as f: + for r in results: + f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") + + # Compute summary + successful = [r for r in results if r.success] + failed = len(results) - len(successful) + retried = [r for r in results if r.retry_count > 0] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + ttfts = [r.first_token_time_ms for r in successful] + tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] + e2es = [r.e2e_latency_ms for r in successful] + itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] + + total_input_tokens = sum(r.input_tokens for r in successful) + total_output_tokens = sum(r.output_tokens for r in successful) + + # Per-port breakdown + per_port_stats: dict[int, dict[str, Any]] = {} + for port in ports: + port_results = [r for r in successful if r.target_port == port] + port_failed = [r for r in results if r.target_port == port and not r.success] + if port_results or port_failed: + port_ttfts = [r.first_token_time_ms for r in port_results] + port_e2es = [r.e2e_latency_ms for r in port_results] + per_port_stats[port] = { + "count": len(port_results), + "failed": len(port_failed), + "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, + "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, + "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, + } + + # Health-check stats + healthy_at_end = await health_checker.healthy_ports + + summary = { + "completed": len(successful), + "failed": failed, + "retried": len(retried), + "duration": duration, + "request_throughput": len(successful) / duration if duration > 0 else 0.0, + "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, + "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, + "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, + "median_ttft_ms": percentile(ttfts, 50), + "p90_ttft_ms": percentile(ttfts, 90), + "p95_ttft_ms": percentile(ttfts, 95), + "p99_ttft_ms": percentile(ttfts, 99), + "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, + "median_tpot_ms": percentile(tpots, 50), + "p90_tpot_ms": percentile(tpots, 90), + "p95_tpot_ms": percentile(tpots, 95), + "p99_tpot_ms": percentile(tpots, 99), + "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, + "median_e2e_latency_ms": percentile(e2es, 50), + "p90_e2e_latency_ms": percentile(e2es, 90), + "p95_e2e_latency_ms": percentile(e2es, 95), + "p99_e2e_latency_ms": percentile(e2es, 99), + "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, + "median_itl_ms": percentile(itls, 50), + "p90_itl_ms": percentile(itls, 90), + "p95_itl_ms": percentile(itls, 95), + "p99_itl_ms": percentile(itls, 99), + "per_port_stats": per_port_stats, + "healthy_ports_at_end": healthy_at_end, + "lb_strategy": lb.strategy, + "num_services": lb.port_count, + } + + # Append summary as the last line + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + + return summary + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30005, + help="Single service port (backward compatible)") + parser.add_argument("--ports", default="", + help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") + parser.add_argument("--model", default="deepseek-v4-flash") + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--input-len", type=int, default=512) + parser.add_argument("--output-len", type=int, default=256) + parser.add_argument("--num-prompts", type=int, default=10) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--lb-strategy", default="round_robin", + choices=["round_robin", "random", "least_conn"], + help="Load balancing strategy across services") + parser.add_argument("--health-check-interval", type=float, default=5.0, + help="Seconds between health checks (default: 5)") + parser.add_argument("--health-max-failures", type=int, default=2, + help="Consecutive failures before marking a port unhealthy (default: 2)") + parser.add_argument("--health-recovery-interval", type=float, default=10.0, + help="Seconds before retrying an unhealthy port (default: 10)") + parser.add_argument("--request-max-retries", type=int, default=2, + help="Max retries per request on failure (default: 2)") + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + + # Determine ports: --ports takes precedence, else fall back to --port. + if args.ports: + ports = [int(p.strip()) for p in args.ports.split(",")] + else: + ports = [args.port] + + health_checker = HealthChecker( + host=args.host, + ports=ports, + check_interval=args.health_check_interval, + max_failures=args.health_max_failures, + recovery_interval=args.health_recovery_interval, + ) + lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) + + summary = asyncio.run(run_benchmark( + host=args.host, + ports=ports, + lb=lb, + health_checker=health_checker, + model=args.model, + concurrency=args.concurrency, + input_len=args.input_len, + output_len=args.output_len, + num_prompts=args.num_prompts, + temperature=args.temperature, + output_file=args.output_file, + )) + + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env new file mode 100644 index 0000000..8f784be --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env @@ -0,0 +1,84 @@ +# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench +# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. +# +# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: +# Service 1: port 30005, GPUs 0,1 +# Service 2: port 30006, GPUs 2,3 +# Service 3: port 30007, GPUs 4,5 +# Service 4: port 30008, GPUs 6,7 +# +# bench_client.py distributes requests across all 4 services via round-robin +# load balancing, giving accurate cluster-wide throughput even at low +# per-service concurrency. + +EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench_no_fp8" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Primary port (backward compatible); multi-service ports are in PORTS. +PORT="30005" +PORTS="30005,30006" + +BACKEND="vllm" +ENGINE="vllm" + +# Native virtual environments on the host. +VENV_SERVER="/data/user1/yy/envs/vllm" +VENV_CLIENT="/data/user1/yy/envs/vllm" + +# Load balancing strategy: round_robin | random | least_conn +LB_STRATEGY="round_robin" + +# Benchmark scenarios: "concurrency input_len output_len num_prompts". +# For multi-service we include low-concurrency scenarios to show how +# requests spread across 2 services. +SCENARIOS=( + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + "1 2048 512 20" + "8 2048 512 160" + "32 2048 512 640" + "128 2048 512 2560" + "1 4096 1024 20" + "8 4096 1024 160" + "32 4096 1024 640" + "128 4096 1024 2560" + "1 8192 1024 20" + "8 8192 1024 160" + "32 8192 1024 640" + "128 8192 1024 2560" + "1 16384 1024 20" + "8 16384 1024 160" + "32 16384 1024 640" + "128 16384 1024 2560" + "1 32768 1024 20" + "8 32768 1024 160" + "32 32768 1024 640" + "128 32768 1024 2560" + "1 65536 1024 20" + "8 65536 1024 160" + "32 65536 1024 640" + "128 65536 1024 2560" + "1 131072 1024 20" + "8 131072 1024 160" + "32 131072 1024 640" + "128 131072 1024 2560" + "1 262144 1024 20" + "8 262144 1024 160" + "32 262144 1024 640" + "128 262144 1024 2560" + "1 524288 1024 20" + "8 524288 1024 160" + "32 524288 1024 640" + "128 524288 1024 2560" +) + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py new file mode 100755 index 0000000..eea1a2e --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. + +Reads JSONL files produced by bench_client.py (one request per line, +last line is the summary) and generates: + - results.json (appended scenarios, following the standard schema) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: + """Read the summary JSON (last line) and all per-request lines.""" + requests: list[dict] = [] + summary: dict | None = None + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The summary line has "completed" and "duration" keys. + if "completed" in obj and "duration" in obj: + summary = obj + else: + requests.append(obj) + return summary, requests + + +def compute_metrics(summary: dict, requests: list[dict]) -> dict: + completed = summary.get("completed", 0) + total = len(requests) + failed = total - completed if total > 0 else 0 + duration_s = summary.get("duration", 0.0) + + # Extract per-request metrics for percentiles. + ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] + tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] + e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] + itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": summary.get("request_throughput", 0.0), + "input_token_throughput": summary.get("input_throughput", 0.0), + "output_token_throughput": summary.get("output_throughput", 0.0), + "total_token_throughput": summary.get("total_throughput", 0.0), + "total_input_tokens": summary.get("total_input_tokens", 0), + "total_output_tokens": summary.get("total_output_tokens", 0), + "e2e_ms": { + "mean": summary.get("mean_e2e_latency_ms", 0.0), + "p50": percentile(e2es, 50), + "p90": percentile(e2es, 90), + "p95": percentile(e2es, 95), + "p99": percentile(e2es, 99), + }, + "ttft_ms": { + "mean": summary.get("mean_ttft_ms", 0.0), + "p50": percentile(ttfts, 50), + "p90": percentile(ttfts, 90), + "p95": percentile(ttfts, 95), + "p99": percentile(ttfts, 99), + }, + "tpot_ms": { + "mean": summary.get("mean_tpot_ms", 0.0), + "p50": percentile(tpots, 50), + "p90": percentile(tpots, 90), + "p95": percentile(tpots, 95), + "p99": percentile(tpots, 99), + }, + "itl_ms": { + "mean": summary.get("mean_itl_ms", 0.0), + "p50": percentile(itls, 50), + "p90": percentile(itls, 90), + "p95": percentile(itls, 95), + "p99": percentile(itls, 99), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") + f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") + f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") + f.write(f"- **Result root**: `{result_root}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + summary, requests = parse_jsonl(jsonl_path) + if summary is None: + continue + + metrics = compute_metrics(summary, requests) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh new file mode 100755 index 0000000..0ec7941 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +# Custom benchmark orchestrator for multi-service vLLM TP=4 on 8x H200. +# +# This runs 2 independent vLLM services (TP=4 each) on GPU groups (0,1,2,3) and (4,5,6,7). +# bench_client.py distributes requests across all 2 services via round-robin LB, +# giving accurate cluster-wide throughput even at low per-service concurrency. +# +# Usage: +# bash run_bench.sh +# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "services=${PORTS}" +log "lb_strategy=${LB_STRATEGY}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 2x vLLM TP=4 multi-service benchmark using bench_client.py (no FP8 KV cache)" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json, sys +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 4, + "num_services": 2, + "ports": "30005,30006", + "gpu_groups": "0,1,2,3|4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", "2 512 256 40", "4 512 256 80", + "8 512 256 160", "16 512 256 320", "32 512 256 640", + "64 512 256 1280", "128 512 256 2560", + "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", + "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# --------------------------------------------------------------------------- +# Server helpers (multi-service) +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +are_all_services_healthy() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + for p in "${PORT_LIST[@]}"; do + if ! is_server_healthy "$p"; then + return 1 + fi + done + return 0 +} + +stop_all_servers() { + log "stopping all vllm services" + # Kill by PID files first. + for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do + [[ -f "$pid_file" ]] || continue + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "killing pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + done + # Fallback: kill any remaining vllm processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + sleep 2 +} + +start_all_servers() { + log "starting all vllm services with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + + if ! are_all_services_healthy "$PORTS"; then + log "error: not all vllm services became healthy" + exit 1 + fi + log "all vllm services are healthy" +} + +# --------------------------------------------------------------------------- +# Pre-benchmark health check with detailed reporting +# --------------------------------------------------------------------------- + +check_services_detailed() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy_count=0 + local unhealthy_ports="" + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + ((healthy_count++)) + else + unhealthy_ports="${unhealthy_ports}${p} " + fi + done + echo "$healthy_count" + if [[ -n "$unhealthy_ports" ]]; then + echo "unhealthy: ${unhealthy_ports}" >&2 + fi +} + +wait_for_all_services() { + local ports="$1" + local max_wait="${2:-240}" + local interval="${3:-5}" + local elapsed=0 + while (( elapsed < max_wait )); do + local healthy_count + healthy_count="$(check_services_detailed "$ports" | head -1)" + IFS=',' read -ra PORT_LIST <<< "$ports" + if (( healthy_count == ${#PORT_LIST[@]} )); then + return 0 + fi + log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" + sleep "$interval" + ((elapsed += interval)) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Mid-benchmark health check (called between scenarios) +# --------------------------------------------------------------------------- + +check_and_report_services() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy=() + local unhealthy=() + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + healthy+=("$p") + else + unhealthy+=("$p") + fi + done + log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" + if [[ ${#unhealthy[@]} -gt 0 ]]; then + log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" + fi + return ${#unhealthy[@]} +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_all_servers + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming services are already running" + if ! wait_for_all_services "$PORTS"; then + log "error: not all healthy services found on ports ${PORTS}" + exit 1 + fi +else + stop_all_servers + start_all_servers +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi + + output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" + + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ + --host 127.0.0.1 \ + --ports "$PORTS" \ + --model "$SERVED_MODEL_NAME" \ + --concurrency "$concurrency" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --lb-strategy "$LB_STRATEGY" \ + --health-check-interval 5 \ + --health-max-failures 2 \ + --health-recovery-interval 10 \ + --request-max-retries 2 \ + --output-file "$output_file" \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + # After failure, check service health before continuing. + check_and_report_services "$PORTS" + continue + } + + log "finished scenario: output=${output_file}" + + # Between scenarios, report service health so we can spot degradation early. + check_and_report_services "$PORTS" || true +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh new file mode 100755 index 0000000..25cce7d --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Start 2 independent vLLM TP=4 services on 8x H200. +# Each service runs on 4 GPUs: (0,1,2,3) and (4,5,6,7). +# This maximizes per-service throughput for higher-concurrency workloads. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_SERVER}" +export PATH="$VENV/bin:$PATH" +VLLM="$VENV/bin/vllm" + +TP=4 +LOG_DIR="${SCRIPT_DIR}/logs" +mkdir -p "$LOG_DIR" + +# 2 services, each on 4 GPUs. +# Port and GPU mapping. +SERVICES=( + "30005:0,1,2,3" + "30006:4,5,6,7" +) + +# Kill any existing vLLM processes for this model. +pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true +sleep 2 + +# Remove old PID files. +rm -f "${SCRIPT_DIR}"/*.pid + +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + GPUS="${svc##*:}" + LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + + echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" + echo "Log: ${LOG}" + + CUDA_VISIBLE_DEVICES="${GPUS}" \ + nohup "$VLLM" serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --block-size 256 \ + --served-model-name "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + > "$LOG" 2>&1 & + PID=$! + echo $PID > "$PID_FILE" + echo "PID: $PID" +done + +echo "" +echo "Waiting for all 2 services to become healthy..." + +MAX_WAIT=240 +all_healthy=0 +for i in $(seq 1 $MAX_WAIT); do + all_healthy=1 + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + all_healthy=0 + break + fi + done + + if [[ "$all_healthy" -eq 1 ]]; then + echo "All 2 services are healthy!" + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + echo " - http://127.0.0.1:${PORT}" + done + exit 0 + fi + + # Check if any process died early. + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + if [[ -f "$PID_FILE" ]]; then + PID="$(cat "$PID_FILE")" + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + tail -200 $LOG 2>/dev/null || true + exit 1 + fi + fi + done + + if (( i % 10 == 0 )); then + echo "Waiting... (${i}/${MAX_WAIT})" + fi + sleep 5 +done + +echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + echo "--- Last 50 lines of port ${PORT} ---" + tail -50 $LOG 2>/dev/null || true +done +exit 1 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/README.md b/experiments/dsv4_h200_vllm_tp8_custom_bench/README.md new file mode 100644 index 0000000..f7717e2 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/README.md @@ -0,0 +1,101 @@ +# dsv4_h200_vllm_tp8_custom_bench + +## 目的 + +当 vLLM 使用 **TP=8**（Tensor Parallel = 8）部署时，单张 H200 上运行 **1 个服务**，独占全部 8 张 GPU： + +| 服务 | 端口 | GPU 组 | +|------|------|--------| +| 1 | 30005 | 0, 1, 2, 3, 4, 5, 6, 7 | + +TP=8 提供最大的单服务 tensor-parallel 宽度，适合极高并发、大 batch 场景。本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API。 + +## 与多服务方案的区别 + +| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | TP=8 (1 服务) | +|------|---------------|---------------|---------------| +| 服务数量 | 4 | 2 | 1 | +| 每服务 GPU | 2 | 4 | 8 | +| 单服务吞吐 | 较低 | 中等 | 最高 | +| 集群总吞吐 | 相近 | 相近 | 相近 | +| 适用场景 | 低并发、低延迟 | 中等并发 | 极高并发、大 batch | + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `bench_client.py` | 自定义异步压测客户端（复用 TP=2/TP=4 版本） | +| `config.env` | 实验配置（单服务、端口 30005） | +| `run_bench.sh` | 编排脚本：启动 1 个服务 → 运行压测 → 解析结果 | +| `start_server.sh` | 启动 1 个 vLLM TP=8 服务（占用全部 8 GPU） | +| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | + +## 快速运行 + +```bash +# 完整流程（自动启动服务、压测、生成报告） +bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh + +# 跳过服务管理（已有服务在运行） +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh + +# 单独运行压测客户端（单服务） +python3 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ + --output-file /tmp/test.jsonl + +# 解析已有结果 +python3 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py \ + experiments/dsv4_h200_vllm_tp8_custom_bench/results/ +``` + +## 场景设计 + +本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： + +| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | +|------|----------|----------|--------|------| +| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | +| 2 | 512 | 256 | 40 | 低并发 | +| 4 | 512 | 256 | 80 | 中低并发 | +| 8 | 512 | 256 | 160 | 中等并发 | +| 16 | 512 | 256 | 320 | 中高并发 | +| 32 | 512 | 256 | 640 | 高并发 | +| 64 | 512 | 256 | 1280 | 很高并发 | +| 128 | 512 | 256 | 2560 | 极限并发，测吞吐上限 | +| 1 | 2048 | 512 | 20 | 长输入基线 | +| 8 | 2048 | 512 | 160 | 长输入中并发 | +| 32 | 2048 | 512 | 640 | 长输入高并发 | +| 128 | 2048 | 512 | 2560 | 长输入极限并发 | +| 1 | 4096 | 1024 | 20 | 超长输入基线 | +| 8 | 4096 | 1024 | 160 | 超长输入中并发 | +| 32 | 4096 | 1024 | 640 | 超长输入高并发 | +| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | + +## 输出格式 + +同 TP=2/TP=4 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 + +## 单独启动/停止服务 + +```bash +# 启动服务 +bash experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh + +# 停止（run_bench.sh 会自动停止，但也可以手动） +port=30005 +pid_file="experiments/dsv4_h200_vllm_tp8_custom_bench/server_port${port}.pid" +[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true +rm -f "$pid_file" +pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true +``` + +## 注意事项 + +- 服务独占全部 8 张 GPU，确保无其他进程占用 GPU。 +- 模型文件路径：`/data/models/DeepSeek-V4-Flash` +- TP=8 相比 TP=2/TP=4 单服务延迟最低，但无法通过多服务并行提升低并发下的集群利用率。 +- 健康检查依赖 `/health` 端点（vLLM 默认启用）。 +- 如果服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py new file mode 100755 index 0000000..d8edc36 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. + +Sends concurrent requests to multiple inference servers and records per-request +latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. + +Designed for TP=2 on 8-GPU setups where 4 independent services run on +(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via +round-robin or random load balancing. Unhealthy services are automatically +skipped with periodic retry. + +Usage: + # Single service (backward compatible) + python3 bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --output-file results.jsonl + + # Multi-service (4 services on ports 30005-30008) + python3 bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --lb-strategy round_robin \ + --output-file results.jsonl + +The output JSONL contains one object per request with raw timing data. +A final summary line ("completed" field) aggregates all requests. +""" + +import argparse +import asyncio +import json +import random +import sys +import time +import urllib.request +from dataclasses import asdict, dataclass, field +from typing import Any + +# Try aiohttp first; fall back to urllib for stdlib-only environments. +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + +@dataclass +class RequestResult: + request_id: int + input_tokens: int = 0 + output_tokens: int = 0 + first_token_time_ms: float = 0.0 # TTFT + e2e_latency_ms: float = 0.0 # total wall time + inter_token_latencies: list[float] = field(default_factory=list) + success: bool = False + error: str = "" + target_port: int = 0 # which service handled this request + retry_count: int = 0 # how many times this request was retried + + @property + def tpot_ms(self) -> float: + """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" + if self.output_tokens <= 1: + return 0.0 + return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) + + @property + def mean_itl_ms(self) -> float: + if not self.inter_token_latencies: + return 0.0 + return sum(self.inter_token_latencies) / len(self.inter_token_latencies) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["tpot_ms"] = self.tpot_ms + d["mean_itl_ms"] = self.mean_itl_ms + return d + + +# --------------------------------------------------------------------------- +# Health checker +# --------------------------------------------------------------------------- + +class HealthChecker: + """Periodically check service health and maintain a healthy-port list.""" + + def __init__( + self, + host: str, + ports: list[int], + check_interval: float = 5.0, + max_failures: int = 2, + recovery_interval: float = 10.0, + ): + self.host = host + self.all_ports = ports + self.check_interval = check_interval + self.max_failures = max_failures + self.recovery_interval = recovery_interval + + # port -> consecutive failure count + self._failures: dict[int, int] = {p: 0 for p in ports} + # port -> last healthy timestamp + self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} + self._lock = asyncio.Lock() + self._task: asyncio.Task | None = None + + async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: + url = f"http://{self.host}:{port}/health" + try: + if HAS_AIOHTTP and session is not None: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + return resp.status == 200 + else: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status == 200 + except Exception: + return False + + async def _check_all(self, session: aiohttp.ClientSession | None) -> None: + checks = [self._check_one(p, session) for p in self.all_ports] + results = await asyncio.gather(*checks, return_exceptions=True) + now = time.monotonic() + async with self._lock: + for port, ok in zip(self.all_ports, results): + if isinstance(ok, Exception): + ok = False + if ok: + self._failures[port] = 0 + self._last_healthy[port] = now + else: + self._failures[port] += 1 + + async def start(self, session: aiohttp.ClientSession | None) -> None: + async def _loop() -> None: + while True: + await asyncio.sleep(self.check_interval) + await self._check_all(session) + self._task = asyncio.create_task(_loop()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + @property + async def healthy_ports(self) -> list[int]: + now = time.monotonic() + async with self._lock: + healthy = [] + for p in self.all_ports: + if self._failures[p] < self.max_failures: + healthy.append(p) + elif now - self._last_healthy[p] > self.recovery_interval: + # Give it another chance after recovery_interval. + self._failures[p] = max(0, self._failures[p] - 1) + healthy.append(p) + return healthy + + async def is_healthy(self, port: int) -> bool: + ports = await self.healthy_ports + return port in ports + + +# --------------------------------------------------------------------------- +# Load balancer with failover +# --------------------------------------------------------------------------- + +class LoadBalancer: + """Distribute requests across healthy backend ports with auto-failover.""" + + def __init__( + self, + ports: list[int], + health_checker: HealthChecker, + strategy: str = "round_robin", + ): + self.all_ports = ports + self.health_checker = health_checker + self.strategy = strategy + self._idx = 0 + self._lock = asyncio.Lock() + + def _next_round_robin(self, healthy: list[int]) -> int: + if not healthy: + return self.all_ports[self._idx % len(self.all_ports)] + for _ in range(len(self.all_ports)): + port = self.all_ports[self._idx % len(self.all_ports)] + self._idx += 1 + if port in healthy: + return port + # Fallback: all unhealthy, pick first healthy anyway. + return healthy[0] if healthy else self.all_ports[0] + + def _next_random(self, healthy: list[int]) -> int: + if healthy: + return random.choice(healthy) + return random.choice(self.all_ports) + + async def next_port(self) -> int: + healthy = await self.health_checker.healthy_ports + async with self._lock: + if self.strategy == "round_robin": + return self._next_round_robin(healthy) + elif self.strategy in ("random", "least_conn"): + return self._next_random(healthy) + else: + return self._next_round_robin(healthy) + + @property + def port_count(self) -> int: + return len(self.all_ports) + + +# --------------------------------------------------------------------------- +# Token helpers +# --------------------------------------------------------------------------- + +def make_prompt(token_len: int, vocab_size: int = 32000) -> str: + """Generate a random-ish prompt of approximately `token_len` tokens.""" + words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", + "dog", "hello", "world", "deep", "seek", "model", "test", + "benchmark", "performance", "latency", "throughput", "token"] + needed = max(token_len, 1) + tokens: list[str] = [] + while len(tokens) < needed: + tokens.extend(words) + return " ".join(tokens[:needed]) + + +# --------------------------------------------------------------------------- +# aiohttp-based async request (preferred) +# --------------------------------------------------------------------------- + +async def send_request_aiohttp( + session: aiohttp.ClientSession, + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + headers = {"Content-Type": "application/json"} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + first_token_received = False + last_token_time = 0.0 + + try: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + text = await resp.text() + result.error = f"HTTP {resp.status}: {text[:200]}" + return result + + async for line in resp.content: + line = line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data_str = line[len("data: "):] + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + content = delta.get("content", "") + if content: + now = time.perf_counter() + if not first_token_received: + result.first_token_time_ms = (now - start_time) * 1000 + first_token_received = True + else: + result.inter_token_latencies.append((now - last_token_time) * 1000) + last_token_time = now + result.output_tokens += 1 + + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.input_tokens = len(prompt.split()) # approximate + result.success = True + + except asyncio.TimeoutError: + result.error = "timeout" + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# urllib-based synchronous request (fallback, no aiohttp) +# --------------------------------------------------------------------------- + +def send_request_urllib( + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + } + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=600) as resp: + body = json.loads(resp.read().decode("utf-8")) + + choices = body.get("choices", []) + if not choices: + result.error = "no choices in response" + return result + + content = choices[0].get("message", {}).get("content", "") + result.output_tokens = len(content.split()) if content else 0 + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.first_token_time_ms = result.e2e_latency_ms + result.input_tokens = len(prompt.split()) + result.success = True + + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# Unified send_request wrapper with retry & failover +# --------------------------------------------------------------------------- + +async def send_request_with_retry( + session: aiohttp.ClientSession | None, + host: str, + lb: LoadBalancer, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, + max_retries: int = 2, +) -> RequestResult: + """Send a request, retrying on failure with a different port each time.""" + result = RequestResult(request_id=request_id) + for attempt in range(max_retries + 1): + port = await lb.next_port() + if HAS_AIOHTTP and session is not None: + result = await send_request_aiohttp( + session, host, port, model, request_id, prompt, max_tokens, temperature + ) + else: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, send_request_urllib, + host, port, model, request_id, prompt, max_tokens, temperature + ) + result.retry_count = attempt + if result.success: + return result + # If failed, try another port on next iteration (unless last attempt). + if attempt < max_retries: + await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish + return result + + +# --------------------------------------------------------------------------- +# Benchmark orchestration +# --------------------------------------------------------------------------- + +async def run_benchmark( + host: str, + ports: list[int], + lb: LoadBalancer, + health_checker: HealthChecker, + model: str, + concurrency: int, + input_len: int, + output_len: int, + num_prompts: int, + temperature: float, + output_file: str, +) -> dict[str, Any]: + """Run the benchmark and write a JSONL file.""" + prompts = [make_prompt(input_len) for _ in range(num_prompts)] + results: list[RequestResult] = [] + semaphore = asyncio.Semaphore(concurrency) + + start = time.perf_counter() + + if HAS_AIOHTTP: + connector = aiohttp.TCPConnector(limit=concurrency * 2) + timeout = aiohttp.ClientTimeout(total=600, sock_read=300) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + await health_checker.start(session) + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + session, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + else: + await health_checker.start(None) + from concurrent.futures import ThreadPoolExecutor + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + None, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + + duration = time.perf_counter() - start + + # Write JSONL + with open(output_file, "w", encoding="utf-8") as f: + for r in results: + f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") + + # Compute summary + successful = [r for r in results if r.success] + failed = len(results) - len(successful) + retried = [r for r in results if r.retry_count > 0] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + ttfts = [r.first_token_time_ms for r in successful] + tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] + e2es = [r.e2e_latency_ms for r in successful] + itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] + + total_input_tokens = sum(r.input_tokens for r in successful) + total_output_tokens = sum(r.output_tokens for r in successful) + + # Per-port breakdown + per_port_stats: dict[int, dict[str, Any]] = {} + for port in ports: + port_results = [r for r in successful if r.target_port == port] + port_failed = [r for r in results if r.target_port == port and not r.success] + if port_results or port_failed: + port_ttfts = [r.first_token_time_ms for r in port_results] + port_e2es = [r.e2e_latency_ms for r in port_results] + per_port_stats[port] = { + "count": len(port_results), + "failed": len(port_failed), + "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, + "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, + "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, + } + + # Health-check stats + healthy_at_end = await health_checker.healthy_ports + + summary = { + "completed": len(successful), + "failed": failed, + "retried": len(retried), + "duration": duration, + "request_throughput": len(successful) / duration if duration > 0 else 0.0, + "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, + "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, + "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, + "median_ttft_ms": percentile(ttfts, 50), + "p90_ttft_ms": percentile(ttfts, 90), + "p95_ttft_ms": percentile(ttfts, 95), + "p99_ttft_ms": percentile(ttfts, 99), + "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, + "median_tpot_ms": percentile(tpots, 50), + "p90_tpot_ms": percentile(tpots, 90), + "p95_tpot_ms": percentile(tpots, 95), + "p99_tpot_ms": percentile(tpots, 99), + "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, + "median_e2e_latency_ms": percentile(e2es, 50), + "p90_e2e_latency_ms": percentile(e2es, 90), + "p95_e2e_latency_ms": percentile(e2es, 95), + "p99_e2e_latency_ms": percentile(e2es, 99), + "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, + "median_itl_ms": percentile(itls, 50), + "p90_itl_ms": percentile(itls, 90), + "p95_itl_ms": percentile(itls, 95), + "p99_itl_ms": percentile(itls, 99), + "per_port_stats": per_port_stats, + "healthy_ports_at_end": healthy_at_end, + "lb_strategy": lb.strategy, + "num_services": lb.port_count, + } + + # Append summary as the last line + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + + return summary + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30005, + help="Single service port (backward compatible)") + parser.add_argument("--ports", default="", + help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") + parser.add_argument("--model", default="deepseek-v4-flash") + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--input-len", type=int, default=512) + parser.add_argument("--output-len", type=int, default=256) + parser.add_argument("--num-prompts", type=int, default=10) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--lb-strategy", default="round_robin", + choices=["round_robin", "random", "least_conn"], + help="Load balancing strategy across services") + parser.add_argument("--health-check-interval", type=float, default=5.0, + help="Seconds between health checks (default: 5)") + parser.add_argument("--health-max-failures", type=int, default=2, + help="Consecutive failures before marking a port unhealthy (default: 2)") + parser.add_argument("--health-recovery-interval", type=float, default=10.0, + help="Seconds before retrying an unhealthy port (default: 10)") + parser.add_argument("--request-max-retries", type=int, default=2, + help="Max retries per request on failure (default: 2)") + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + + # Determine ports: --ports takes precedence, else fall back to --port. + if args.ports: + ports = [int(p.strip()) for p in args.ports.split(",")] + else: + ports = [args.port] + + health_checker = HealthChecker( + host=args.host, + ports=ports, + check_interval=args.health_check_interval, + max_failures=args.health_max_failures, + recovery_interval=args.health_recovery_interval, + ) + lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) + + summary = asyncio.run(run_benchmark( + host=args.host, + ports=ports, + lb=lb, + health_checker=health_checker, + model=args.model, + concurrency=args.concurrency, + input_len=args.input_len, + output_len=args.output_len, + num_prompts=args.num_prompts, + temperature=args.temperature, + output_file=args.output_file, + )) + + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env new file mode 100644 index 0000000..0cc0836 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env @@ -0,0 +1,83 @@ +# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench +# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. +# +# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: +# Service 1: port 30005, GPUs 0,1 +# Service 2: port 30006, GPUs 2,3 +# Service 3: port 30007, GPUs 4,5 +# Service 4: port 30008, GPUs 6,7 +# +# bench_client.py distributes requests across all 4 services via round-robin +# load balancing, giving accurate cluster-wide throughput even at low +# per-service concurrency. + +EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Primary port (backward compatible); multi-service ports are in PORTS. +PORT="30005" +PORTS="30005" + +BACKEND="vllm" +ENGINE="vllm" + +# Native virtual environments on the host. +VENV_SERVER="/data/user1/yy/envs/vllm" +VENV_CLIENT="/data/user1/yy/envs/vllm" + +# Load balancing strategy: round_robin | random | least_conn +LB_STRATEGY="round_robin" + +# Benchmark scenarios: "concurrency input_len output_len num_prompts". +# For single-service TP=8 we cover standard concurrency gradients. +SCENARIOS=( + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + "1 2048 512 20" + "8 2048 512 160" + "32 2048 512 640" + "128 2048 512 2560" + "1 4096 1024 20" + "8 4096 1024 160" + "32 4096 1024 640" + "128 4096 1024 2560" + "1 8192 1024 20" + "8 8192 1024 160" + "32 8192 1024 640" + "128 8192 1024 2560" + "1 16384 1024 20" + "8 16384 1024 160" + "32 16384 1024 640" + "128 16384 1024 2560" + "1 32768 1024 20" + "8 32768 1024 160" + "32 32768 1024 640" + "128 32768 1024 2560" + "1 65536 1024 20" + "8 65536 1024 160" + "32 65536 1024 640" + "128 65536 1024 2560" + "1 131072 1024 20" + "8 131072 1024 160" + "32 131072 1024 640" + "128 131072 1024 2560" + "1 262144 1024 20" + "8 262144 1024 160" + "32 262144 1024 640" + "128 262144 1024 2560" + "1 524288 1024 20" + "8 524288 1024 160" + "32 524288 1024 640" + "128 524288 1024 2560" +) + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py b/experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py new file mode 100755 index 0000000..eea1a2e --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. + +Reads JSONL files produced by bench_client.py (one request per line, +last line is the summary) and generates: + - results.json (appended scenarios, following the standard schema) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: + """Read the summary JSON (last line) and all per-request lines.""" + requests: list[dict] = [] + summary: dict | None = None + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The summary line has "completed" and "duration" keys. + if "completed" in obj and "duration" in obj: + summary = obj + else: + requests.append(obj) + return summary, requests + + +def compute_metrics(summary: dict, requests: list[dict]) -> dict: + completed = summary.get("completed", 0) + total = len(requests) + failed = total - completed if total > 0 else 0 + duration_s = summary.get("duration", 0.0) + + # Extract per-request metrics for percentiles. + ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] + tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] + e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] + itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": summary.get("request_throughput", 0.0), + "input_token_throughput": summary.get("input_throughput", 0.0), + "output_token_throughput": summary.get("output_throughput", 0.0), + "total_token_throughput": summary.get("total_throughput", 0.0), + "total_input_tokens": summary.get("total_input_tokens", 0), + "total_output_tokens": summary.get("total_output_tokens", 0), + "e2e_ms": { + "mean": summary.get("mean_e2e_latency_ms", 0.0), + "p50": percentile(e2es, 50), + "p90": percentile(e2es, 90), + "p95": percentile(e2es, 95), + "p99": percentile(e2es, 99), + }, + "ttft_ms": { + "mean": summary.get("mean_ttft_ms", 0.0), + "p50": percentile(ttfts, 50), + "p90": percentile(ttfts, 90), + "p95": percentile(ttfts, 95), + "p99": percentile(ttfts, 99), + }, + "tpot_ms": { + "mean": summary.get("mean_tpot_ms", 0.0), + "p50": percentile(tpots, 50), + "p90": percentile(tpots, 90), + "p95": percentile(tpots, 95), + "p99": percentile(tpots, 99), + }, + "itl_ms": { + "mean": summary.get("mean_itl_ms", 0.0), + "p50": percentile(itls, 50), + "p90": percentile(itls, 90), + "p95": percentile(itls, 95), + "p99": percentile(itls, 99), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") + f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") + f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") + f.write(f"- **Result root**: `{result_root}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + summary, requests = parse_jsonl(jsonl_path) + if summary is None: + continue + + metrics = compute_metrics(summary, requests) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh new file mode 100755 index 0000000..9120605 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# Custom benchmark orchestrator for single-service vLLM TP=8 on 8x H200. +# +# This runs 1 vLLM service (TP=8) on all 8 GPUs (0,1,2,3,4,5,6,7). +# bench_client.py sends requests to the single service. +# +# Usage: +# bash run_bench.sh +# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "services=${PORTS}" +log "lb_strategy=${LB_STRATEGY}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 1x vLLM TP=8 single-service benchmark using bench_client.py" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json, sys +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "num_services": 1, + "ports": "30005", + "gpu_groups": "0,1,2,3,4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", "2 512 256 40", "4 512 256 80", + "8 512 256 160", "16 512 256 320", "32 512 256 640", + "64 512 256 1280", "128 512 256 2560", + "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", + "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# --------------------------------------------------------------------------- +# Server helpers (single-service) +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +are_all_services_healthy() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + for p in "${PORT_LIST[@]}"; do + if ! is_server_healthy "$p"; then + return 1 + fi + done + return 0 +} + +stop_all_servers() { + log "stopping all vllm services" + # Kill by PID files first. + for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do + [[ -f "$pid_file" ]] || continue + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "killing pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + done + # Fallback: kill any remaining vllm processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + sleep 2 +} + +start_all_servers() { + log "starting all vllm services with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + + if ! are_all_services_healthy "$PORTS"; then + log "error: not all vllm services became healthy" + exit 1 + fi + log "all vllm services are healthy" +} + +# --------------------------------------------------------------------------- +# Pre-benchmark health check with detailed reporting +# --------------------------------------------------------------------------- + +check_services_detailed() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy_count=0 + local unhealthy_ports="" + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + ((healthy_count++)) + else + unhealthy_ports="${unhealthy_ports}${p} " + fi + done + echo "$healthy_count" + if [[ -n "$unhealthy_ports" ]]; then + echo "unhealthy: ${unhealthy_ports}" >&2 + fi +} + +wait_for_all_services() { + local ports="$1" + local max_wait="${2:-240}" + local interval="${3:-5}" + local elapsed=0 + while (( elapsed < max_wait )); do + local healthy_count + healthy_count="$(check_services_detailed "$ports" | head -1)" + IFS=',' read -ra PORT_LIST <<< "$ports" + if (( healthy_count == ${#PORT_LIST[@]} )); then + return 0 + fi + log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" + sleep "$interval" + ((elapsed += interval)) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Mid-benchmark health check (called between scenarios) +# --------------------------------------------------------------------------- + +check_and_report_services() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy=() + local unhealthy=() + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + healthy+=("$p") + else + unhealthy+=("$p") + fi + done + log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" + if [[ ${#unhealthy[@]} -gt 0 ]]; then + log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" + fi + return ${#unhealthy[@]} +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_all_servers + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming services are already running" + if ! wait_for_all_services "$PORTS"; then + log "error: not all healthy services found on ports ${PORTS}" + exit 1 + fi +else + stop_all_servers + start_all_servers +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi + + output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" + + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ + --host 127.0.0.1 \ + --port "$PORT" \ + --model "$SERVED_MODEL_NAME" \ + --concurrency "$concurrency" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --output-file "$output_file" \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + # After failure, check service health before continuing. + check_and_report_services "$PORTS" + continue + } + + log "finished scenario: output=${output_file}" + + # Between scenarios, report service health so we can spot degradation early. + check_and_report_services "$PORTS" || true +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh new file mode 100755 index 0000000..90b030b --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Start 1 vLLM TP=8 service on 8x H200. +# Uses all 8 GPUs: (0,1,2,3,4,5,6,7). +# This maximizes single-service throughput for highest-concurrency workloads. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_SERVER}" +export PATH="$VENV/bin:$PATH" +VLLM="$VENV/bin/vllm" + +TP=8 +LOG_DIR="${SCRIPT_DIR}/logs" +mkdir -p "$LOG_DIR" + +# 1 service on all 8 GPUs. +# Port and GPU mapping. +SERVICES=( + "30005:0,1,2,3,4,5,6,7" +) + +# Kill any existing vLLM processes for this model. +pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true +sleep 2 + +# Remove old PID files. +rm -f "${SCRIPT_DIR}"/*.pid + +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + GPUS="${svc##*:}" + LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + + echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" + echo "Log: ${LOG}" + + CUDA_VISIBLE_DEVICES="${GPUS}" \ + nohup "$VLLM" serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --block-size 256 \ + --served-model-name "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + > "$LOG" 2>&1 & + PID=$! + echo $PID > "$PID_FILE" + echo "PID: $PID" +done + +echo "" +echo "Waiting for service to become healthy..." + +MAX_WAIT=240 +all_healthy=0 +for i in $(seq 1 $MAX_WAIT); do + all_healthy=1 + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + all_healthy=0 + break + fi + done + + if [[ "$all_healthy" -eq 1 ]]; then + echo "Service is healthy!" + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + echo " - http://127.0.0.1:${PORT}" + done + exit 0 + fi + + # Check if any process died early. + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + if [[ -f "$PID_FILE" ]]; then + PID="$(cat "$PID_FILE")" + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + tail -200 $LOG 2>/dev/null || true + exit 1 + fi + fi + done + + if (( i % 10 == 0 )); then + echo "Waiting... (${i}/${MAX_WAIT})" + fi + sleep 5 +done + +echo "ERROR: Service did not become healthy after ${MAX_WAIT} retries" +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + echo "--- Last 50 lines of port ${PORT} ---" + tail -50 $LOG 2>/dev/null || true +done +exit 1 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md new file mode 100644 index 0000000..fc5aa6f --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md @@ -0,0 +1,103 @@ +# dsv4_h200_vllm_tp8_custom_bench_no_fp8 + +> **Note:** This variant does NOT use `--kv-cache-dtype fp8`. + +## 目的 + +当 vLLM 使用 **TP=8**（Tensor Parallel = 8）部署时，单张 H200 上运行 **1 个服务**，独占全部 8 张 GPU： + +| 服务 | 端口 | GPU 组 | +|------|------|--------| +| 1 | 30005 | 0, 1, 2, 3, 4, 5, 6, 7 | + +TP=8 提供最大的单服务 tensor-parallel 宽度，适合极高并发、大 batch 场景。本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API。 + +## 与多服务方案的区别 + +| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | TP=8 (1 服务) | +|------|---------------|---------------|---------------| +| 服务数量 | 4 | 2 | 1 | +| 每服务 GPU | 2 | 4 | 8 | +| 单服务吞吐 | 较低 | 中等 | 最高 | +| 集群总吞吐 | 相近 | 相近 | 相近 | +| 适用场景 | 低并发、低延迟 | 中等并发 | 极高并发、大 batch | + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `bench_client.py` | 自定义异步压测客户端（复用 TP=2/TP=4 版本） | +| `config.env` | 实验配置（单服务、端口 30005） | +| `run_bench.sh` | 编排脚本：启动 1 个服务 → 运行压测 → 解析结果 | +| `start_server.sh` | 启动 1 个 vLLM TP=8 服务（占用全部 8 GPU） | +| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | + +## 快速运行 + +```bash +# 完整流程（自动启动服务、压测、生成报告） +bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh + +# 跳过服务管理（已有服务在运行） +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh + +# 单独运行压测客户端（单服务） +python3 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ + --output-file /tmp/test.jsonl + +# 解析已有结果 +python3 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py \ + experiments/dsv4_h200_vllm_tp8_custom_bench/results/ +``` + +## 场景设计 + +本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： + +| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | +|------|----------|----------|--------|------| +| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | +| 2 | 512 | 256 | 40 | 低并发 | +| 4 | 512 | 256 | 80 | 中低并发 | +| 8 | 512 | 256 | 160 | 中等并发 | +| 16 | 512 | 256 | 320 | 中高并发 | +| 32 | 512 | 256 | 640 | 高并发 | +| 64 | 512 | 256 | 1280 | 很高并发 | +| 128 | 512 | 256 | 2560 | 极限并发，测吞吐上限 | +| 1 | 2048 | 512 | 20 | 长输入基线 | +| 8 | 2048 | 512 | 160 | 长输入中并发 | +| 32 | 2048 | 512 | 640 | 长输入高并发 | +| 128 | 2048 | 512 | 2560 | 长输入极限并发 | +| 1 | 4096 | 1024 | 20 | 超长输入基线 | +| 8 | 4096 | 1024 | 160 | 超长输入中并发 | +| 32 | 4096 | 1024 | 640 | 超长输入高并发 | +| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | + +## 输出格式 + +同 TP=2/TP=4 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 + +## 单独启动/停止服务 + +```bash +# 启动服务 +bash experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh + +# 停止（run_bench.sh 会自动停止，但也可以手动） +port=30005 +pid_file="experiments/dsv4_h200_vllm_tp8_custom_bench/server_port${port}.pid" +[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true +rm -f "$pid_file" +pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true +``` + +## 注意事项 + +- 服务独占全部 8 张 GPU，确保无其他进程占用 GPU。 +- 模型文件路径：`/data/models/DeepSeek-V4-Flash` +- TP=8 相比 TP=2/TP=4 单服务延迟最低，但无法通过多服务并行提升低并发下的集群利用率。 +- 健康检查依赖 `/health` 端点（vLLM 默认启用）。 +- 如果服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py new file mode 100755 index 0000000..d8edc36 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. + +Sends concurrent requests to multiple inference servers and records per-request +latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. + +Designed for TP=2 on 8-GPU setups where 4 independent services run on +(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via +round-robin or random load balancing. Unhealthy services are automatically +skipped with periodic retry. + +Usage: + # Single service (backward compatible) + python3 bench_client.py \ + --host 127.0.0.1 --port 30005 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --output-file results.jsonl + + # Multi-service (4 services on ports 30005-30008) + python3 bench_client.py \ + --host 127.0.0.1 \ + --ports 30005,30006,30007,30008 \ + --model deepseek-v4-flash \ + --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ + --lb-strategy round_robin \ + --output-file results.jsonl + +The output JSONL contains one object per request with raw timing data. +A final summary line ("completed" field) aggregates all requests. +""" + +import argparse +import asyncio +import json +import random +import sys +import time +import urllib.request +from dataclasses import asdict, dataclass, field +from typing import Any + +# Try aiohttp first; fall back to urllib for stdlib-only environments. +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + +@dataclass +class RequestResult: + request_id: int + input_tokens: int = 0 + output_tokens: int = 0 + first_token_time_ms: float = 0.0 # TTFT + e2e_latency_ms: float = 0.0 # total wall time + inter_token_latencies: list[float] = field(default_factory=list) + success: bool = False + error: str = "" + target_port: int = 0 # which service handled this request + retry_count: int = 0 # how many times this request was retried + + @property + def tpot_ms(self) -> float: + """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" + if self.output_tokens <= 1: + return 0.0 + return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) + + @property + def mean_itl_ms(self) -> float: + if not self.inter_token_latencies: + return 0.0 + return sum(self.inter_token_latencies) / len(self.inter_token_latencies) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["tpot_ms"] = self.tpot_ms + d["mean_itl_ms"] = self.mean_itl_ms + return d + + +# --------------------------------------------------------------------------- +# Health checker +# --------------------------------------------------------------------------- + +class HealthChecker: + """Periodically check service health and maintain a healthy-port list.""" + + def __init__( + self, + host: str, + ports: list[int], + check_interval: float = 5.0, + max_failures: int = 2, + recovery_interval: float = 10.0, + ): + self.host = host + self.all_ports = ports + self.check_interval = check_interval + self.max_failures = max_failures + self.recovery_interval = recovery_interval + + # port -> consecutive failure count + self._failures: dict[int, int] = {p: 0 for p in ports} + # port -> last healthy timestamp + self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} + self._lock = asyncio.Lock() + self._task: asyncio.Task | None = None + + async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: + url = f"http://{self.host}:{port}/health" + try: + if HAS_AIOHTTP and session is not None: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + return resp.status == 200 + else: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status == 200 + except Exception: + return False + + async def _check_all(self, session: aiohttp.ClientSession | None) -> None: + checks = [self._check_one(p, session) for p in self.all_ports] + results = await asyncio.gather(*checks, return_exceptions=True) + now = time.monotonic() + async with self._lock: + for port, ok in zip(self.all_ports, results): + if isinstance(ok, Exception): + ok = False + if ok: + self._failures[port] = 0 + self._last_healthy[port] = now + else: + self._failures[port] += 1 + + async def start(self, session: aiohttp.ClientSession | None) -> None: + async def _loop() -> None: + while True: + await asyncio.sleep(self.check_interval) + await self._check_all(session) + self._task = asyncio.create_task(_loop()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + @property + async def healthy_ports(self) -> list[int]: + now = time.monotonic() + async with self._lock: + healthy = [] + for p in self.all_ports: + if self._failures[p] < self.max_failures: + healthy.append(p) + elif now - self._last_healthy[p] > self.recovery_interval: + # Give it another chance after recovery_interval. + self._failures[p] = max(0, self._failures[p] - 1) + healthy.append(p) + return healthy + + async def is_healthy(self, port: int) -> bool: + ports = await self.healthy_ports + return port in ports + + +# --------------------------------------------------------------------------- +# Load balancer with failover +# --------------------------------------------------------------------------- + +class LoadBalancer: + """Distribute requests across healthy backend ports with auto-failover.""" + + def __init__( + self, + ports: list[int], + health_checker: HealthChecker, + strategy: str = "round_robin", + ): + self.all_ports = ports + self.health_checker = health_checker + self.strategy = strategy + self._idx = 0 + self._lock = asyncio.Lock() + + def _next_round_robin(self, healthy: list[int]) -> int: + if not healthy: + return self.all_ports[self._idx % len(self.all_ports)] + for _ in range(len(self.all_ports)): + port = self.all_ports[self._idx % len(self.all_ports)] + self._idx += 1 + if port in healthy: + return port + # Fallback: all unhealthy, pick first healthy anyway. + return healthy[0] if healthy else self.all_ports[0] + + def _next_random(self, healthy: list[int]) -> int: + if healthy: + return random.choice(healthy) + return random.choice(self.all_ports) + + async def next_port(self) -> int: + healthy = await self.health_checker.healthy_ports + async with self._lock: + if self.strategy == "round_robin": + return self._next_round_robin(healthy) + elif self.strategy in ("random", "least_conn"): + return self._next_random(healthy) + else: + return self._next_round_robin(healthy) + + @property + def port_count(self) -> int: + return len(self.all_ports) + + +# --------------------------------------------------------------------------- +# Token helpers +# --------------------------------------------------------------------------- + +def make_prompt(token_len: int, vocab_size: int = 32000) -> str: + """Generate a random-ish prompt of approximately `token_len` tokens.""" + words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", + "dog", "hello", "world", "deep", "seek", "model", "test", + "benchmark", "performance", "latency", "throughput", "token"] + needed = max(token_len, 1) + tokens: list[str] = [] + while len(tokens) < needed: + tokens.extend(words) + return " ".join(tokens[:needed]) + + +# --------------------------------------------------------------------------- +# aiohttp-based async request (preferred) +# --------------------------------------------------------------------------- + +async def send_request_aiohttp( + session: aiohttp.ClientSession, + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + headers = {"Content-Type": "application/json"} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + first_token_received = False + last_token_time = 0.0 + + try: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + text = await resp.text() + result.error = f"HTTP {resp.status}: {text[:200]}" + return result + + async for line in resp.content: + line = line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data_str = line[len("data: "):] + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + content = delta.get("content", "") + if content: + now = time.perf_counter() + if not first_token_received: + result.first_token_time_ms = (now - start_time) * 1000 + first_token_received = True + else: + result.inter_token_latencies.append((now - last_token_time) * 1000) + last_token_time = now + result.output_tokens += 1 + + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.input_tokens = len(prompt.split()) # approximate + result.success = True + + except asyncio.TimeoutError: + result.error = "timeout" + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# urllib-based synchronous request (fallback, no aiohttp) +# --------------------------------------------------------------------------- + +def send_request_urllib( + host: str, + port: int, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, +) -> RequestResult: + """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" + url = f"http://{host}:{port}/v1/chat/completions" + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + } + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} + + result = RequestResult(request_id=request_id, target_port=port) + start_time = time.perf_counter() + + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=600) as resp: + body = json.loads(resp.read().decode("utf-8")) + + choices = body.get("choices", []) + if not choices: + result.error = "no choices in response" + return result + + content = choices[0].get("message", {}).get("content", "") + result.output_tokens = len(content.split()) if content else 0 + result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 + result.first_token_time_ms = result.e2e_latency_ms + result.input_tokens = len(prompt.split()) + result.success = True + + except Exception as e: + result.error = str(e)[:200] + + return result + + +# --------------------------------------------------------------------------- +# Unified send_request wrapper with retry & failover +# --------------------------------------------------------------------------- + +async def send_request_with_retry( + session: aiohttp.ClientSession | None, + host: str, + lb: LoadBalancer, + model: str, + request_id: int, + prompt: str, + max_tokens: int, + temperature: float, + max_retries: int = 2, +) -> RequestResult: + """Send a request, retrying on failure with a different port each time.""" + result = RequestResult(request_id=request_id) + for attempt in range(max_retries + 1): + port = await lb.next_port() + if HAS_AIOHTTP and session is not None: + result = await send_request_aiohttp( + session, host, port, model, request_id, prompt, max_tokens, temperature + ) + else: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, send_request_urllib, + host, port, model, request_id, prompt, max_tokens, temperature + ) + result.retry_count = attempt + if result.success: + return result + # If failed, try another port on next iteration (unless last attempt). + if attempt < max_retries: + await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish + return result + + +# --------------------------------------------------------------------------- +# Benchmark orchestration +# --------------------------------------------------------------------------- + +async def run_benchmark( + host: str, + ports: list[int], + lb: LoadBalancer, + health_checker: HealthChecker, + model: str, + concurrency: int, + input_len: int, + output_len: int, + num_prompts: int, + temperature: float, + output_file: str, +) -> dict[str, Any]: + """Run the benchmark and write a JSONL file.""" + prompts = [make_prompt(input_len) for _ in range(num_prompts)] + results: list[RequestResult] = [] + semaphore = asyncio.Semaphore(concurrency) + + start = time.perf_counter() + + if HAS_AIOHTTP: + connector = aiohttp.TCPConnector(limit=concurrency * 2) + timeout = aiohttp.ClientTimeout(total=600, sock_read=300) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + await health_checker.start(session) + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + session, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + else: + await health_checker.start(None) + from concurrent.futures import ThreadPoolExecutor + + async def bounded_send(idx: int, prompt: str) -> RequestResult: + async with semaphore: + return await send_request_with_retry( + None, host, lb, model, idx, prompt, output_len, temperature + ) + + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] + results = await asyncio.gather(*tasks) + await health_checker.stop() + + duration = time.perf_counter() - start + + # Write JSONL + with open(output_file, "w", encoding="utf-8") as f: + for r in results: + f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") + + # Compute summary + successful = [r for r in results if r.success] + failed = len(results) - len(successful) + retried = [r for r in results if r.retry_count > 0] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + ttfts = [r.first_token_time_ms for r in successful] + tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] + e2es = [r.e2e_latency_ms for r in successful] + itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] + + total_input_tokens = sum(r.input_tokens for r in successful) + total_output_tokens = sum(r.output_tokens for r in successful) + + # Per-port breakdown + per_port_stats: dict[int, dict[str, Any]] = {} + for port in ports: + port_results = [r for r in successful if r.target_port == port] + port_failed = [r for r in results if r.target_port == port and not r.success] + if port_results or port_failed: + port_ttfts = [r.first_token_time_ms for r in port_results] + port_e2es = [r.e2e_latency_ms for r in port_results] + per_port_stats[port] = { + "count": len(port_results), + "failed": len(port_failed), + "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, + "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, + "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, + } + + # Health-check stats + healthy_at_end = await health_checker.healthy_ports + + summary = { + "completed": len(successful), + "failed": failed, + "retried": len(retried), + "duration": duration, + "request_throughput": len(successful) / duration if duration > 0 else 0.0, + "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, + "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, + "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, + "median_ttft_ms": percentile(ttfts, 50), + "p90_ttft_ms": percentile(ttfts, 90), + "p95_ttft_ms": percentile(ttfts, 95), + "p99_ttft_ms": percentile(ttfts, 99), + "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, + "median_tpot_ms": percentile(tpots, 50), + "p90_tpot_ms": percentile(tpots, 90), + "p95_tpot_ms": percentile(tpots, 95), + "p99_tpot_ms": percentile(tpots, 99), + "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, + "median_e2e_latency_ms": percentile(e2es, 50), + "p90_e2e_latency_ms": percentile(e2es, 90), + "p95_e2e_latency_ms": percentile(e2es, 95), + "p99_e2e_latency_ms": percentile(e2es, 99), + "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, + "median_itl_ms": percentile(itls, 50), + "p90_itl_ms": percentile(itls, 90), + "p95_itl_ms": percentile(itls, 95), + "p99_itl_ms": percentile(itls, 99), + "per_port_stats": per_port_stats, + "healthy_ports_at_end": healthy_at_end, + "lb_strategy": lb.strategy, + "num_services": lb.port_count, + } + + # Append summary as the last line + with open(output_file, "a", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + + return summary + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30005, + help="Single service port (backward compatible)") + parser.add_argument("--ports", default="", + help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") + parser.add_argument("--model", default="deepseek-v4-flash") + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--input-len", type=int, default=512) + parser.add_argument("--output-len", type=int, default=256) + parser.add_argument("--num-prompts", type=int, default=10) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--lb-strategy", default="round_robin", + choices=["round_robin", "random", "least_conn"], + help="Load balancing strategy across services") + parser.add_argument("--health-check-interval", type=float, default=5.0, + help="Seconds between health checks (default: 5)") + parser.add_argument("--health-max-failures", type=int, default=2, + help="Consecutive failures before marking a port unhealthy (default: 2)") + parser.add_argument("--health-recovery-interval", type=float, default=10.0, + help="Seconds before retrying an unhealthy port (default: 10)") + parser.add_argument("--request-max-retries", type=int, default=2, + help="Max retries per request on failure (default: 2)") + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + + # Determine ports: --ports takes precedence, else fall back to --port. + if args.ports: + ports = [int(p.strip()) for p in args.ports.split(",")] + else: + ports = [args.port] + + health_checker = HealthChecker( + host=args.host, + ports=ports, + check_interval=args.health_check_interval, + max_failures=args.health_max_failures, + recovery_interval=args.health_recovery_interval, + ) + lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) + + summary = asyncio.run(run_benchmark( + host=args.host, + ports=ports, + lb=lb, + health_checker=health_checker, + model=args.model, + concurrency=args.concurrency, + input_len=args.input_len, + output_len=args.output_len, + num_prompts=args.num_prompts, + temperature=args.temperature, + output_file=args.output_file, + )) + + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env new file mode 100644 index 0000000..618099b --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env @@ -0,0 +1,83 @@ +# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench +# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. +# +# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: +# Service 1: port 30005, GPUs 0,1 +# Service 2: port 30006, GPUs 2,3 +# Service 3: port 30007, GPUs 4,5 +# Service 4: port 30008, GPUs 6,7 +# +# bench_client.py distributes requests across all 4 services via round-robin +# load balancing, giving accurate cluster-wide throughput even at low +# per-service concurrency. + +EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench_no_fp8" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Primary port (backward compatible); multi-service ports are in PORTS. +PORT="30005" +PORTS="30005" + +BACKEND="vllm" +ENGINE="vllm" + +# Native virtual environments on the host. +VENV_SERVER="/data/user1/yy/envs/vllm" +VENV_CLIENT="/data/user1/yy/envs/vllm" + +# Load balancing strategy: round_robin | random | least_conn +LB_STRATEGY="round_robin" + +# Benchmark scenarios: "concurrency input_len output_len num_prompts". +# For single-service TP=8 we cover standard concurrency gradients. +SCENARIOS=( + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + "1 2048 512 20" + "8 2048 512 160" + "32 2048 512 640" + "128 2048 512 2560" + "1 4096 1024 20" + "8 4096 1024 160" + "32 4096 1024 640" + "128 4096 1024 2560" + "1 8192 1024 20" + "8 8192 1024 160" + "32 8192 1024 640" + "128 8192 1024 2560" + "1 16384 1024 20" + "8 16384 1024 160" + "32 16384 1024 640" + "128 16384 1024 2560" + "1 32768 1024 20" + "8 32768 1024 160" + "32 32768 1024 640" + "128 32768 1024 2560" + "1 65536 1024 20" + "8 65536 1024 160" + "32 65536 1024 640" + "128 65536 1024 2560" + "1 131072 1024 20" + "8 131072 1024 160" + "32 131072 1024 640" + "128 131072 1024 2560" + "1 262144 1024 20" + "8 262144 1024 160" + "32 262144 1024 640" + "128 262144 1024 2560" + "1 524288 1024 20" + "8 524288 1024 160" + "32 524288 1024 640" + "128 524288 1024 2560" +) + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py new file mode 100755 index 0000000..eea1a2e --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. + +Reads JSONL files produced by bench_client.py (one request per line, +last line is the summary) and generates: + - results.json (appended scenarios, following the standard schema) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: + """Read the summary JSON (last line) and all per-request lines.""" + requests: list[dict] = [] + summary: dict | None = None + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The summary line has "completed" and "duration" keys. + if "completed" in obj and "duration" in obj: + summary = obj + else: + requests.append(obj) + return summary, requests + + +def compute_metrics(summary: dict, requests: list[dict]) -> dict: + completed = summary.get("completed", 0) + total = len(requests) + failed = total - completed if total > 0 else 0 + duration_s = summary.get("duration", 0.0) + + # Extract per-request metrics for percentiles. + ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] + tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] + e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] + itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] + + def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(s) - 1) + return s[f] + (k - f) * (s[c] - s[f]) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": summary.get("request_throughput", 0.0), + "input_token_throughput": summary.get("input_throughput", 0.0), + "output_token_throughput": summary.get("output_throughput", 0.0), + "total_token_throughput": summary.get("total_throughput", 0.0), + "total_input_tokens": summary.get("total_input_tokens", 0), + "total_output_tokens": summary.get("total_output_tokens", 0), + "e2e_ms": { + "mean": summary.get("mean_e2e_latency_ms", 0.0), + "p50": percentile(e2es, 50), + "p90": percentile(e2es, 90), + "p95": percentile(e2es, 95), + "p99": percentile(e2es, 99), + }, + "ttft_ms": { + "mean": summary.get("mean_ttft_ms", 0.0), + "p50": percentile(ttfts, 50), + "p90": percentile(ttfts, 90), + "p95": percentile(ttfts, 95), + "p99": percentile(ttfts, 99), + }, + "tpot_ms": { + "mean": summary.get("mean_tpot_ms", 0.0), + "p50": percentile(tpots, 50), + "p90": percentile(tpots, 90), + "p95": percentile(tpots, 95), + "p99": percentile(tpots, 99), + }, + "itl_ms": { + "mean": summary.get("mean_itl_ms", 0.0), + "p50": percentile(itls, 50), + "p90": percentile(itls, 90), + "p95": percentile(itls, 95), + "p99": percentile(itls, 99), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") + f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") + f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") + f.write(f"- **Result root**: `{result_root}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + summary, requests = parse_jsonl(jsonl_path) + if summary is None: + continue + + metrics = compute_metrics(summary, requests) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh new file mode 100755 index 0000000..2319cae --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# Custom benchmark orchestrator for single-service vLLM TP=8 on 8x H200. +# +# This runs 1 vLLM service (TP=8) on all 8 GPUs (0,1,2,3,4,5,6,7). +# bench_client.py sends requests to the single service. +# +# Usage: +# bash run_bench.sh +# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "services=${PORTS}" +log "lb_strategy=${LB_STRATEGY}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 1x vLLM TP=8 single-service (no FP8 KV cache) benchmark using bench_client.py" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json, sys +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "num_services": 1, + "ports": "30005", + "gpu_groups": "0,1,2,3,4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", "2 512 256 40", "4 512 256 80", + "8 512 256 160", "16 512 256 320", "32 512 256 640", + "64 512 256 1280", "128 512 256 2560", + "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", + "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" + ] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# --------------------------------------------------------------------------- +# Server helpers (single-service) +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +are_all_services_healthy() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + for p in "${PORT_LIST[@]}"; do + if ! is_server_healthy "$p"; then + return 1 + fi + done + return 0 +} + +stop_all_servers() { + log "stopping all vllm services" + # Kill by PID files first. + for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do + [[ -f "$pid_file" ]] || continue + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "killing pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + done + # Fallback: kill any remaining vllm processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + sleep 2 +} + +start_all_servers() { + log "starting all vllm services with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + + if ! are_all_services_healthy "$PORTS"; then + log "error: not all vllm services became healthy" + exit 1 + fi + log "all vllm services are healthy" +} + +# --------------------------------------------------------------------------- +# Pre-benchmark health check with detailed reporting +# --------------------------------------------------------------------------- + +check_services_detailed() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy_count=0 + local unhealthy_ports="" + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + ((healthy_count++)) + else + unhealthy_ports="${unhealthy_ports}${p} " + fi + done + echo "$healthy_count" + if [[ -n "$unhealthy_ports" ]]; then + echo "unhealthy: ${unhealthy_ports}" >&2 + fi +} + +wait_for_all_services() { + local ports="$1" + local max_wait="${2:-240}" + local interval="${3:-5}" + local elapsed=0 + while (( elapsed < max_wait )); do + local healthy_count + healthy_count="$(check_services_detailed "$ports" | head -1)" + IFS=',' read -ra PORT_LIST <<< "$ports" + if (( healthy_count == ${#PORT_LIST[@]} )); then + return 0 + fi + log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" + sleep "$interval" + ((elapsed += interval)) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Mid-benchmark health check (called between scenarios) +# --------------------------------------------------------------------------- + +check_and_report_services() { + local ports="$1" + IFS=',' read -ra PORT_LIST <<< "$ports" + local healthy=() + local unhealthy=() + for p in "${PORT_LIST[@]}"; do + if is_server_healthy "$p"; then + healthy+=("$p") + else + unhealthy+=("$p") + fi + done + log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" + if [[ ${#unhealthy[@]} -gt 0 ]]; then + log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" + fi + return ${#unhealthy[@]} +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_all_servers + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming services are already running" + if ! wait_for_all_services "$PORTS"; then + log "error: not all healthy services found on ports ${PORTS}" + exit 1 + fi +else + stop_all_servers + start_all_servers +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + # Fallback for legacy 3-field scenarios. + if [[ -z "${num_prompts:-}" ]]; then + num_prompts="$NUM_PROMPTS" + fi + + output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" + + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ + --host 127.0.0.1 \ + --port "$PORT" \ + --model "$SERVED_MODEL_NAME" \ + --concurrency "$concurrency" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --output-file "$output_file" \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + # After failure, check service health before continuing. + check_and_report_services "$PORTS" + continue + } + + log "finished scenario: output=${output_file}" + + # Between scenarios, report service health so we can spot degradation early. + check_and_report_services "$PORTS" || true +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh new file mode 100755 index 0000000..b9dad2f --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Start 1 vLLM TP=8 service on 8x H200. +# Uses all 8 GPUs: (0,1,2,3,4,5,6,7). +# This maximizes single-service throughput for highest-concurrency workloads. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_SERVER}" +export PATH="$VENV/bin:$PATH" +VLLM="$VENV/bin/vllm" + +TP=8 +LOG_DIR="${SCRIPT_DIR}/logs" +mkdir -p "$LOG_DIR" + +# 1 service on all 8 GPUs. +# Port and GPU mapping. +SERVICES=( + "30005:0,1,2,3,4,5,6,7" +) + +# Kill any existing vLLM processes for this model. +pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true +sleep 2 + +# Remove old PID files. +rm -f "${SCRIPT_DIR}"/*.pid + +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + GPUS="${svc##*:}" + LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + + echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" + echo "Log: ${LOG}" + + CUDA_VISIBLE_DEVICES="${GPUS}" \ + nohup "$VLLM" serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --block-size 256 \ + --served-model-name "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + > "$LOG" 2>&1 & + PID=$! + echo $PID > "$PID_FILE" + echo "PID: $PID" +done + +echo "" +echo "Waiting for service to become healthy..." + +MAX_WAIT=240 +all_healthy=0 +for i in $(seq 1 $MAX_WAIT); do + all_healthy=1 + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + all_healthy=0 + break + fi + done + + if [[ "$all_healthy" -eq 1 ]]; then + echo "Service is healthy!" + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + echo " - http://127.0.0.1:${PORT}" + done + exit 0 + fi + + # Check if any process died early. + for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" + if [[ -f "$PID_FILE" ]]; then + PID="$(cat "$PID_FILE")" + if ! kill -0 "$PID" 2>/dev/null; then + echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + tail -200 $LOG 2>/dev/null || true + exit 1 + fi + fi + done + + if (( i % 10 == 0 )); then + echo "Waiting... (${i}/${MAX_WAIT})" + fi + sleep 5 +done + +echo "ERROR: Service did not become healthy after ${MAX_WAIT} retries" +for svc in "${SERVICES[@]}"; do + PORT="${svc%%:*}" + LOG="${LOG_DIR}/server_port${PORT}_*.log" + echo "--- Last 50 lines of port ${PORT} ---" + tail -50 $LOG 2>/dev/null || true +done +exit 1 From d783143ddbb9acaa9dcec8e6989840873fcb8e93 Mon Sep 17 00:00:00 2001 From: Root User Date: Fri, 10 Jul 2026 08:44:38 +0000 Subject: [PATCH 39/53] vllm tp_dp_matrix: dockerize deployment and fix missing DOCKER_CLIENT_IMAGE export - Run vLLM server inside vllm/vllm-openai:latest container on H200. - Run benchmark client inside lmsysorg/sglang:latest using sglang.bench_serving --backend vllm, matching the sglang reference. - Use RUNTIME_BASE for self-contained logs/pids/tmp instead of hardcoded /data/user1/yy paths. - Update model path to /data3/hf_models/DeepSeek-V4-Flash. - Sample only low/high concurrency endpoints to match sglang matrix. - Fix docker 'invalid reference format' caused by DOCKER_CLIENT_IMAGE not being exported to bash -c subshells used by run_bench_serving. - Add offline dummy ShareGPT dataset for random benchmark mode. --- .../dsv4_h200_vllm_tp_dp_matrix/compare.py | 10 +- .../dsv4_h200_vllm_tp_dp_matrix/config.env | 50 +++++++--- .../generate_scenarios.py | 21 ++-- .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 99 ++++++++++++++----- .../start_vllm_docker.sh | 29 ++++-- .../start_vllm_dp.sh | 71 +++++++------ 6 files changed, 177 insertions(+), 103 deletions(-) diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py index bdd3224..b8d259b 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py @@ -22,10 +22,10 @@ def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, ttft_ok = ttft_p95_ms < ttft_limit_ms tpot_ok = tpot_mean_ms < tpot_limit_ms if ttft_ok and tpot_ok: - return "✅" + return "PASS" if ttft_ok or tpot_ok: - return "⚠️" - return "❌" + return "PARTIAL" + return "FAIL" def gpu_memory_str(gpu: dict | None) -> str: @@ -82,7 +82,7 @@ def main(): f.write("## Summary\n\n") f.write(f"- Model: `{model}`\n") f.write(f"- Hardware: {hardware}\n") - f.write("- Backend: vLLM\n") + f.write("- Backend: vLLM (Docker)\n") f.write("- Benchmark client: `sglang.bench_serving`\n") f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") @@ -152,7 +152,7 @@ def main(): f.write("\n## Notes\n\n") f.write("- SLO check uses TTFT P95 and TPOT mean.\n") - f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 61b8be2..246c1e1 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -1,4 +1,4 @@ -# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs). +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using vLLM. # Tests vLLM with three parallel configurations: # TP=2, DP=4 -> 2 GPUs per replica, 4 replicas # TP=4, DP=2 -> 4 GPUs per replica, 2 replicas @@ -6,18 +6,25 @@ EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" +MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" VLLM_PORT="${VLLM_PORT:-30030}" -VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" -VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" -# The benchmark client is sglang.bench_serving, even when the backend is vLLM. -VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + # Parallel configurations to test. Format: "TP DP" declare -a PARALLEL_CONFIGS=( "2 4" @@ -25,14 +32,34 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) +# vLLM server settings +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +BLOCK_SIZE="${BLOCK_SIZE:-256}" + +# Deployment switch. 0 = native vllm venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" + +# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving, +# so the client runs inside the SGLang image and targets the vLLM backend via the +# OpenAI-compatible API. +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H200 nodes. +DATASET_PATH="${DATASET_PATH:-${SCRIPT_DIR}/runtime/dummy_sharegpt.json}" + # Matrix and concurrency rules are defined in matrix.json by default. -# This file is consumed by generate_scenarios.py. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" # Sampling density for concurrency. -# 0 = use the default heuristic in generate_scenarios.py (6-8 points). -# 2 = only test the two endpoints (low and high). +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" # Per-scenario timeout to avoid hangs (seconds). @@ -47,8 +74,3 @@ DRY_RUN="${DRY_RUN:-0}" # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. GRID_LIMIT="${GRID_LIMIT:-0}" - -# Docker deployment switch. If 1, vLLM is launched via Docker using the image -# below instead of the local VENV_VLLM environment. -USE_DOCKER="${USE_DOCKER:-0}" -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py index ef46e01..f722479 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py @@ -15,23 +15,16 @@ from pathlib import Path def sample_concurrency(low: int, high: int, target: int) -> list[int]: - """Return a sorted list of concurrency values in [low, high]. + """Return only the low and high concurrency values in [low, high]. - If the range is small, return every integer. Otherwise sample roughly - `target` points linearly between low and high (inclusive). + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. """ assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" - if high - low + 1 <= target: - return list(range(low, high + 1)) - - points = set() - points.add(low) - points.add(high) - step = (high - low) / (target - 1) - for i in range(1, target - 1): - v = low + round(step * i) - points.add(max(low, min(high, v))) - return sorted(points) + if low == high: + return [low] + return [low, high] def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index 99e8de3..f004020 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM. +# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM (Docker). set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -12,6 +12,10 @@ source "${SCRIPT_DIR}/../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE \ + DOCKER_IMAGE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT + RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" @@ -21,7 +25,14 @@ GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" DRY_RUN="${DRY_RUN:-0}" GRID_LIMIT="${GRID_LIMIT:-0}" -PYTHON="${VENV_CLIENT}/bin/python" +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" mkdir -p "$log_dir_global" @@ -42,7 +53,7 @@ stop_server() { local tp="$1" local dp="$2" - local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" if [[ -f "$pid_file" ]]; then local pid pid="$(cat "$pid_file")" @@ -55,6 +66,9 @@ stop_server() { rm -f "$pid_file" fi + # Fallback: remove any Docker container started by this experiment. + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + # Fallback: kill any vllm serve processes for this model. pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true @@ -68,10 +82,14 @@ build_server_args() { local args=( "vllm serve" "$MODEL_PATH" --trust-remote-code - --kv-cache-dtype fp8 - --block-size 256 + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune + --host 0.0.0.0 --port "$VLLM_PORT" ) if [[ "$dp" -gt 1 ]]; then @@ -106,20 +124,51 @@ restart_server() { start_server "$tp" "$dp" } +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" && -f "${DATASET_PATH}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") + docker run --rm \ + --network host \ + "${vol_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e HF_DATASETS_OFFLINE=1 \ + "${DOCKER_CLIENT_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} +export -f run_bench_serving + run_warmup() { local input_len="$1" local output_len="$2" log "warming up (input=${input_len}, output=${output_len}, num=1)" - "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ - --backend vllm \ - --host 127.0.0.1 \ - --port "$VLLM_PORT" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num 1 \ - --env-python "$PYTHON" \ - >> "${log_dir_global}/warmup.log" 2>&1 + bash -c ' + run_bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "'"$VLLM_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' log "warmup completed" } @@ -307,7 +356,7 @@ run_parallel_config() { "$ACCELERATOR" \ "$CHIP" \ "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_VLLM" \ + "$DOCKER_IMAGE" \ "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" local server_args_str @@ -383,20 +432,22 @@ run_parallel_config() { gpu_pid="$(start_gpu_monitor "$gpu_csv")" bench_rc=0 - timeout "$SCENARIO_TIMEOUT_S" \ - "$PYTHON" -m sglang.bench_serving \ + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ --backend vllm \ --host 127.0.0.1 \ - --port "$VLLM_PORT" \ + --port "'"$VLLM_PORT"'" \ --dataset-name random \ - --random-input-len "$isl" \ - --random-output-len "$dsl" \ - --num-prompts "$num" \ - --max-concurrency "$conc" \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$dsl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ --request-rate 10000 \ - --output-file "$output_file" \ + --output-file "'"$output_file"'" \ --output-details \ - > "$detail_log" 2>&1 || bench_rc=$? + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? stop_gpu_monitor "$gpu_pid" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh index ae7f728..e133fd7 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -2,7 +2,7 @@ # Start vLLM server in Docker for a given TPxDP configuration. # Usage: start_vllm_docker.sh # -# Uses the vllm/vllm-openai image and the same minimal argument set as the +# Uses the vllm/vllm-openai image and the same argument set as the # bare-metal start script. The container is removed automatically on stop. set -e @@ -13,15 +13,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -cd /data/user1/yy -mkdir -p logs +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" PORT="${VLLM_PORT:-30030}" NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" -PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" -LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" rm -f "$PID_FILE" # Clean up any stale container with the same name. @@ -30,10 +30,14 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( serve "$MODEL_PATH" --trust-remote-code - --kv-cache-dtype bfloat16 - --block-size 256 + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune + --host 0.0.0.0 --port "$PORT" ) @@ -50,7 +54,7 @@ echo "Image: $IMAGE" echo "Model: $MODEL_PATH" echo "Container name: $NAME" echo "Host port: $PORT" -echo "Command: vllm $SERVER_ARGS_STR" +echo "Command: vllm ${SERVER_ARGS_STR}" echo "Log: $LOG" # Run docker in the foreground so that killing the host process stops the @@ -59,12 +63,17 @@ echo "Log: $LOG" nohup docker run --rm \ --name "$NAME" \ --gpus all \ + --ipc host \ + --shm-size 16g \ + --ulimit memlock=-1 \ --entrypoint vllm \ -p "${PORT}:${PORT}" \ - -v /data/models:/data/models:ro \ - -v /data/user1/yy/tmp:/tmp \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ "$IMAGE" \ "${SERVER_ARGS[@]}" \ > "$LOG" 2>&1 & diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh index 77b662a..741ca9c 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -2,14 +2,9 @@ # Start vLLM server for a given TP×DP configuration. # Usage: start_vllm_dp.sh # -# Uses the minimal argument set from the official DeepSeek-V4-Flash recipe: -# --trust-remote-code -# --kv-cache-dtype fp8 -# --block-size 256 -# --tensor-parallel-size -# --no-enable-flashinfer-autotune -# When DP > 1, only --data-parallel-size is added so vLLM uses its internal -# DP load balancer (single API server, single port). +# By default this delegates to the Docker start script because the experiment +# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the +# local VENV_VLLM environment instead. set -e TP="${1}" @@ -19,38 +14,42 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -if [[ "${USE_DOCKER:-0}" == "1" ]]; then +if [[ "${USE_DOCKER:-1}" == "1" ]]; then exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" fi -cd /data/user1/yy -mkdir -p logs +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" VENV="${VENV_VLLM}" export PATH="$VENV/bin:$PATH" export PYTHONUNBUFFERED=1 -export TMPDIR=/data/user1/yy/tmp +export TMPDIR="${RUNTIME_BASE}/tmp" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" -LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" SERVER_ARGS=( - vllm serve "$MODEL_PATH" - --trust-remote-code - --kv-cache-dtype fp8 - --block-size 256 - --tensor-parallel-size "$TP" - --no-enable-flashinfer-autotune - --port "$VLLM_PORT" + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" ) if [[ "$DP" -gt 1 ]]; then - SERVER_ARGS+=( - --data-parallel-size "$DP" - ) + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) fi SERVER_ARGS_STR="${SERVER_ARGS[*]}" @@ -69,18 +68,18 @@ echo "PID: $PID" echo "Waiting for health on port ${VLLM_PORT}..." for i in $(seq 1 240); do - if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then - echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: vLLM server exited early" - tail -200 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 done echo "ERROR: vLLM server not healthy after 240 retries" From 5e95cee1a0d9a7007263c856d63387eea4269d56 Mon Sep 17 00:00:00 2001 From: Root User Date: Sat, 11 Jul 2026 06:11:19 +0000 Subject: [PATCH 40/53] bench: add adaptive concurrency search --- experiments/ADAPTIVE_CONCURRENCY_USAGE.md | 128 ++++ .../README_adaptive_concurrency.md | 109 +++ .../adaptive_config.env | 49 ++ .../dsv4_h200_sglang_tp_dp_matrix/config.env | 4 +- .../run_adaptive_concurrency.sh | 181 +++++ .../README_adaptive_concurrency.md | 115 +++ .../adaptive_config.env | 49 ++ .../dsv4_h200_vllm_tp_dp_matrix/config.env | 2 +- .../run_adaptive_concurrency.sh | 164 +++++ scripts/common/adaptive_bench_lib.sh | 682 ++++++++++++++++++ scripts/common/adaptive_concurrency.py | 320 ++++++++ 11 files changed, 1800 insertions(+), 3 deletions(-) create mode 100644 experiments/ADAPTIVE_CONCURRENCY_USAGE.md create mode 100644 experiments/dsv4_h200_sglang_tp_dp_matrix/README_adaptive_concurrency.md create mode 100644 experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/README_adaptive_concurrency.md create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100755 scripts/common/adaptive_bench_lib.sh create mode 100755 scripts/common/adaptive_concurrency.py diff --git a/experiments/ADAPTIVE_CONCURRENCY_USAGE.md b/experiments/ADAPTIVE_CONCURRENCY_USAGE.md new file mode 100644 index 0000000..4e28428 --- /dev/null +++ b/experiments/ADAPTIVE_CONCURRENCY_USAGE.md @@ -0,0 +1,128 @@ +# 自适应并发 Benchmark 使用说明 + +这套脚本对每个固定的 `(TP, DP, ISL, DSL)` 依次测试 +`C=1,2,4,8,...`，直到 Total TPS 连续两次增长不足 2%，或达到并发上限、 +发生 OOM。每个正式测点发送 `C * 5` 个请求，并先用相同并发 C warm-up。 + +## 1. 进入目录 + +vLLM： + +```bash +cd /data3/yy/sskj/experiments/dsv4_h200_vllm_tp_dp_matrix +``` + +SGLang： + +```bash +cd /data3/yy/sskj/experiments/dsv4_h200_sglang_tp_dp_matrix +``` + +两个目录的入口都是： + +```bash +bash run_adaptive_concurrency.sh +``` + +## 2. 先 dry-run + +只打印 TP/DP、ISL/DSL 和并发搜索计划，不加载模型： + +```bash +DRY_RUN=1 bash run_adaptive_concurrency.sh +``` + +## 3. 单组合 smoke + +只测 `TP=8, DP=1, ISL=1K, DSL=128`，并发搜索到 8： + +```bash +RUN_ID=smoke-$(date +%Y%m%d-%H%M%S) \ +TP_LIST="8" \ +ISL_LIST="1024" \ +DSL_LIST="128" \ +GRID_LIMIT=1 \ +SEARCH_MAX_CONCURRENCY=8 \ +RESULT_BASE=/data3/yy/sskj_adaptive_results/smoke \ +bash run_adaptive_concurrency.sh +``` + +## 4. 正式运行 + +长任务放进 tmux。vLLM 示例： + +```bash +tmux new-session -d -s vllm-adaptive \ + "cd /data3/yy/sskj/experiments/dsv4_h200_vllm_tp_dp_matrix && \ + RUN_ID=vllm-\$(date +%Y%m%d-%H%M%S) \ + RESULT_BASE=/data3/yy/sskj_adaptive_results/vllm \ + bash run_adaptive_concurrency.sh" +``` + +SGLang 示例： + +```bash +tmux new-session -d -s sglang-adaptive \ + "cd /data3/yy/sskj/experiments/dsv4_h200_sglang_tp_dp_matrix && \ + RUN_ID=sglang-\$(date +%Y%m%d-%H%M%S) \ + RESULT_BASE=/data3/yy/sskj_adaptive_results/sglang \ + bash run_adaptive_concurrency.sh" +``` + +查看任务： + +```bash +tmux ls +tmux attach -t vllm-adaptive +tmux attach -t sglang-adaptive +``` + +从 tmux 退出但保持任务运行：按 `Ctrl-b`，松开，再按 `d`。 + +## 5. 常用覆盖参数 + +无需修改脚本，可以把变量写在命令前： + +```bash +MODEL_PATH=/path/to/model \ +DATASET_PATH=/path/to/ShareGPT.json \ +DOCKER_IMAGE=your/server:image \ +DOCKER_CLIENT_IMAGE=lmsysorg/sglang:latest \ +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ +SEARCH_MAX_CONCURRENCY=512 \ +TPS_MIN_GAIN_PCT=2.0 \ +PLATEAU_PATIENCE=2 \ +bash run_adaptive_concurrency.sh +``` + +- vLLM 端口用 `VLLM_PORT`，SGLang 端口用 `SGLANG_PORT`。 +- `TP_LIST="8"` 只跑 TP=8；DP 使用 `config.env` 中对应的 TP/DP 组合。 +- `ISL_LIST="1024 4096"`、`DSL_LIST="128 256"` 用来筛选长度。 +- `GRID_LIMIT=1` 表示每组 TP/DP 只跑一个 ISL/DSL shape。 +- `BENCH_WARMUP_MAX_REQUESTS=0` 表示每个测点用完整并发 C warm-up；正数表示上限。 +- 正式基线保持 `BENCH_DATASET_NAME=random` 和 `RANDOM_RANGE_RATIO=1.0`。 +- `random-ids` 只用于检查链路，不与正式 ShareGPT-backed 结果混用。 + +换机器或模型时，至少检查 `MODEL_PATH`、`DATASET_PATH`、Docker 镜像、GPU 编号、 +端口、模型最大上下文和显存参数。需要永久修改时再编辑对应目录的 `config.env`。 + +## 6. 结果 + +每次运行在 `${RESULT_BASE}/${RUN_ID}/` 下生成： + +```text +adaptive_points.csv 每个并发测点的 TPS、TTFT、TPOT 等 +adaptive_summary.csv 每个 ISL/DSL/TP/DP 的饱和并发和最佳 TPS 并发 +adaptive_summary.md 可直接阅读的汇总表 +run_manifest.json 本次模型、数据集和搜索参数 +logs/ 编排与服务日志 +tp*_dp*/raw_outputs/ sglang.bench_serving 原始 JSON +tp*_dp*/gpu_logs/ nvidia-smi 采样 +``` + +快速查看： + +```bash +column -s, -t adaptive_summary.csv | less -S +tail -f logs/orchestrator.log +``` diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/README_adaptive_concurrency.md b/experiments/dsv4_h200_sglang_tp_dp_matrix/README_adaptive_concurrency.md new file mode 100644 index 0000000..0545572 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/README_adaptive_concurrency.md @@ -0,0 +1,109 @@ +# SGLang Adaptive Concurrency Search + +`run_adaptive_concurrency.sh` keeps the existing ISL/DSL and TP/DP matrix, but +searches concurrency at runtime instead of testing only low/high endpoints. + +## Search rule + +For every fixed `(TP, DP, ISL, DSL)`: + +```text +C = 1, 2, 4, 8, ... up to SEARCH_MAX_CONCURRENCY +num_prompts = C * NUM_PROMPTS_MULTIPLIER +``` + +The search stops when Total TPS grows by less than `TPS_MIN_GAIN_PCT` for +`PLATEAU_PATIENCE` consecutive points. Defaults are 2% and two points. + +- `saturation_concurrency`: first point in the final low-gain streak +- `best_tps_concurrency`: tested point with the highest Total TPS +- `stop_probe_concurrency`: last point tested to confirm the stop + +OOM stops only the current shape. The service is restarted before the next +shape. A non-OOM failure is retried once after a service restart. + +Before every measured point, the client sends `C` concurrent warm-up requests. +This keeps lazy kernel compilation and CUDA graph capture out of the measured +TTFT/TPS. `BENCH_WARMUP_MAX_REQUESTS=0` means no cap; use a positive value only +when deliberately limiting warm-up at very high concurrency. + +## Dataset requirement + +The default remains compatible with the fixed matrix baseline: + +```text +--dataset-name random --dataset-path DATASET_PATH +--random-range-ratio 1.0 +``` + +In the installed SGLang client, the upstream default `0.0` samples each length +uniformly from `1` to the requested maximum. This adaptive experiment sets +`1.0` explicitly so every point represents the fixed ISL/DSL shown in the +matrix. Override `RANDOM_RANGE_RATIO` only when a mixed-length workload is +intentional. + +`DATASET_PATH` must have at least +`SEARCH_MAX_CONCURRENCY * NUM_PROMPTS_MULTIPLIER` usable two-turn +conversations. The configured dataset is checked before model loading. + +For setup smoke tests without ShareGPT, explicitly use: + +```bash +BENCH_DATASET_NAME=random-ids +``` + +This sends generated token IDs with `--tokenize-prompt`; it is convenient for +smoke tests but changes the token distribution, so do not mix it with random +ShareGPT-backed baseline results. + +## Commands + +Print the plan without starting a server: + +```bash +DRY_RUN=1 bash run_adaptive_concurrency.sh +``` + +Run one small search: + +```bash +TP_LIST="8" \ +ISL_LIST="1024" \ +DSL_LIST="128" \ +GRID_LIMIT=1 \ +SEARCH_MAX_CONCURRENCY=8 \ +DATASET_PATH=/path/to/ShareGPT.json \ +bash run_adaptive_concurrency.sh +``` + +Persistent-container mode remains supported: + +```bash +bash start_sglang_container.sh +CONTAINER_NAME="dsv4_h200_sglang_tp_dp_matrix_sglang_container" \ +DATASET_PATH=/path/to/ShareGPT.json \ +bash run_adaptive_concurrency.sh +``` + +Run long searches inside tmux. + +## Outputs + +Results are written to `adaptive_results//`: + +```text +adaptive_points.jsonl +adaptive_points.csv +adaptive_summary.jsonl +adaptive_summary.csv +adaptive_summary.md +run_manifest.json +tp*/raw_outputs/ +tp*/metrics/ +tp*/logs/ +tp*/gpu_logs/ +``` + +Every successful point is accepted only when `completed == num_prompts` and +actual input/output lengths match the requested shape within configured +tolerances. Invalid workloads never participate in saturation decisions. diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..4085397 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, DSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/DSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +DSL_LIST="${DSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env index d5ac289..bddd299 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env @@ -6,7 +6,7 @@ EXPERIMENT="dsv4_h200_sglang_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" +MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30031}" @@ -45,7 +45,7 @@ DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" # Dataset used by sglang.bench_serving --dataset-name random. # The random sampler needs a ShareGPT-style JSON file locally; it falls back to # downloading from HuggingFace, which usually fails on offline H200 nodes. -DATASET_PATH="${DATASET_PATH:-/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json}" +DATASET_PATH="${DATASET_PATH:-/data3/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..90f7827 --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local dsl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$dsl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/README_adaptive_concurrency.md b/experiments/dsv4_h200_vllm_tp_dp_matrix/README_adaptive_concurrency.md new file mode 100644 index 0000000..d6a3ced --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/README_adaptive_concurrency.md @@ -0,0 +1,115 @@ +# vLLM Adaptive Concurrency Search + +`run_adaptive_concurrency.sh` keeps the existing ISL/DSL and TP/DP matrix, but +searches concurrency at runtime instead of testing only low/high endpoints. + +## Search rule + +For every fixed `(TP, DP, ISL, DSL)`: + +```text +C = 1, 2, 4, 8, ... up to SEARCH_MAX_CONCURRENCY +num_prompts = C * NUM_PROMPTS_MULTIPLIER +``` + +The search stops when Total TPS grows by less than `TPS_MIN_GAIN_PCT` for +`PLATEAU_PATIENCE` consecutive points. Defaults are 2% and two points. + +- `saturation_concurrency`: first point in the final low-gain streak +- `best_tps_concurrency`: tested point with the highest Total TPS +- `stop_probe_concurrency`: last point tested to confirm the stop + +OOM stops only the current shape. The service is restarted before the next +shape. A non-OOM failure is retried once after a service restart. + +Before every measured point, the client sends `C` concurrent warm-up requests. +This keeps lazy kernel compilation and CUDA graph capture out of the measured +TTFT/TPS. `BENCH_WARMUP_MAX_REQUESTS=0` means no cap; use a positive value only +when deliberately limiting warm-up at very high concurrency. + +## Dataset requirement + +The default remains compatible with the fixed matrix baseline: + +```text +--dataset-name random --dataset-path DATASET_PATH +--random-range-ratio 1.0 +``` + +In the installed SGLang client, the upstream default `0.0` samples each length +uniformly from `1` to the requested maximum. This adaptive experiment sets +`1.0` explicitly so every point represents the fixed ISL/DSL shown in the +matrix. Override `RANDOM_RANGE_RATIO` only when a mixed-length workload is +intentional. + +`DATASET_PATH` must have at least +`SEARCH_MAX_CONCURRENCY * NUM_PROMPTS_MULTIPLIER` usable two-turn +conversations. The current `runtime/dummy_sharegpt.json` has only two and is +not valid for a real adaptive run. + +For setup smoke tests without ShareGPT, explicitly use: + +```bash +BENCH_DATASET_NAME=random-ids +``` + +This sends generated token IDs with `--tokenize-prompt`; it is convenient for +smoke tests but changes the token distribution, so do not mix it with random +ShareGPT-backed baseline results. + +## Commands + +Print the plan without starting a server: + +```bash +DRY_RUN=1 bash run_adaptive_concurrency.sh +``` + +Run one small search: + +```bash +TP_LIST="8" \ +ISL_LIST="1024" \ +DSL_LIST="128" \ +GRID_LIMIT=1 \ +SEARCH_MAX_CONCURRENCY=8 \ +DATASET_PATH=/path/to/ShareGPT.json \ +bash run_adaptive_concurrency.sh +``` + +Run the full search in tmux: + +```bash +tmux new -s vllm-adaptive +DATASET_PATH=/path/to/ShareGPT.json bash run_adaptive_concurrency.sh +``` + +Tune the stop rule when needed: + +```bash +TPS_MIN_GAIN_PCT=3.0 \ +PLATEAU_PATIENCE=2 \ +SEARCH_MAX_CONCURRENCY=512 \ +bash run_adaptive_concurrency.sh +``` + +## Outputs + +Results are written to `adaptive_results//`: + +```text +adaptive_points.jsonl +adaptive_points.csv +adaptive_summary.jsonl +adaptive_summary.csv +adaptive_summary.md +run_manifest.json +tp*/raw_outputs/ +tp*/metrics/ +tp*/logs/ +tp*/gpu_logs/ +``` + +Every successful point is accepted only when `completed == num_prompts` and +actual input/output lengths match the requested shape within configured +tolerances. Invalid workloads never participate in saturation decisions. diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..4085397 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, DSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/DSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +DSL_LIST="${DSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 246c1e1..7644eaf 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -51,7 +51,7 @@ DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" # Dataset used by sglang.bench_serving --dataset-name random. # The random sampler needs a ShareGPT-style JSON file locally; it falls back to # downloading from HuggingFace, which usually fails on offline H200 nodes. -DATASET_PATH="${DATASET_PATH:-${SCRIPT_DIR}/runtime/dummy_sharegpt.json}" +DATASET_PATH="${DATASET_PATH:-/data3/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..8b1fdd0 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/DSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local dsl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$dsl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh new file mode 100755 index 0000000..3f8b158 --- /dev/null +++ b/scripts/common/adaptive_bench_lib.sh @@ -0,0 +1,682 @@ +#!/usr/bin/env bash +# Shared adaptive-concurrency search loop for serving benchmarks. + +ADAPTIVE_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ADAPTIVE_HELPER="${ADAPTIVE_COMMON_DIR}/adaptive_concurrency.py" + +adaptive_timestamp() { + date --iso-8601=seconds +} + +adaptive_json_number_or_null() { + local value="${1:-}" + if [[ -n "$value" ]]; then + printf '%s' "$value" + else + printf 'null' + fi +} + +adaptive_warmup_request_count() { + local concurrency="$1" + if (( BENCH_WARMUP_MAX_REQUESTS > 0 && concurrency > BENCH_WARMUP_MAX_REQUESTS )); then + printf '%s' "$BENCH_WARMUP_MAX_REQUESTS" + else + printf '%s' "$concurrency" + fi +} +export -f adaptive_warmup_request_count + +adaptive_validate_config() { + local name + for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY SEARCH_MULTIPLIER \ + NUM_PROMPTS_MULTIPLIER PLATEAU_PATIENCE MAX_POINT_RETRIES \ + BENCH_WARMUP_MAX_REQUESTS; do + if ! [[ "${!name}" =~ ^[0-9]+$ ]]; then + log "ERROR: ${name} must be a non-negative integer, got ${!name}" + return 1 + fi + done + if (( SEARCH_START_CONCURRENCY < 1 )); then + log "ERROR: SEARCH_START_CONCURRENCY must be at least 1" + return 1 + fi + if (( SEARCH_MAX_CONCURRENCY < SEARCH_START_CONCURRENCY )); then + log "ERROR: SEARCH_MAX_CONCURRENCY must be >= SEARCH_START_CONCURRENCY" + return 1 + fi + if (( SEARCH_MULTIPLIER < 2 )); then + log "ERROR: SEARCH_MULTIPLIER must be at least 2" + return 1 + fi + if (( NUM_PROMPTS_MULTIPLIER < 1 || PLATEAU_PATIENCE < 1 )); then + log "ERROR: NUM_PROMPTS_MULTIPLIER and PLATEAU_PATIENCE must be at least 1" + return 1 + fi + if ! [[ "$TPS_MIN_GAIN_PCT" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + log "ERROR: TPS_MIN_GAIN_PCT must be a non-negative number" + return 1 + fi + if ! [[ "$RANDOM_RANGE_RATIO" =~ ^[0-9]+([.][0-9]+)?$ ]] || \ + ! awk -v ratio="$RANDOM_RANGE_RATIO" 'BEGIN { exit !(ratio >= 0 && ratio <= 1) }'; then + log "ERROR: RANDOM_RANGE_RATIO must be between 0.0 and 1.0" + return 1 + fi +} + +adaptive_start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +adaptive_stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +adaptive_warmup() { + local log_path="$1" + log "running synthetic warmup" + timeout "$SCENARIO_TIMEOUT_S" bash -c \ + 'engine_run_bench 1024 128 1 1 /dev/null' \ + >> "$log_path" 2>&1 +} + +adaptive_restart_server() { + local tp="$1" + local dp="$2" + log "restarting ${ENGINE} server tp=${tp} dp=${dp}" + engine_stop_server "$tp" "$dp" + sleep "$SERVER_RESTART_COOLDOWN_S" + engine_start_server "$tp" "$dp" + adaptive_warmup "${ADAPTIVE_LOG_DIR}/${ENGINE}_tp${tp}_dp${dp}.warmup.log" +} + +adaptive_append_point_from_metrics() { + local metrics_file="$1" + local tp="$2" + local dp="$3" + local mark="$4" + local isl="$5" + local dsl="$6" + local concurrency="$7" + local num_prompts="$8" + local attempt="$9" + local gain_pct="${10:-}" + local plateau_streak="${11:-0}" + local raw_file="${12}" + local detail_log="${13}" + local gain_json + gain_json="$(adaptive_json_number_or_null "$gain_pct")" + + jq -c \ + --arg timestamp "$(adaptive_timestamp)" \ + --arg engine "$ENGINE" \ + --argjson tp "$tp" \ + --argjson dp "$dp" \ + --arg mark "$mark" \ + --argjson isl "$isl" \ + --argjson dsl "$dsl" \ + --argjson concurrency "$concurrency" \ + --argjson num_prompts "$num_prompts" \ + --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ + --argjson attempt "$attempt" \ + --argjson gain_pct "$gain_json" \ + --argjson plateau_streak "$plateau_streak" \ + --arg raw_file "$raw_file" \ + --arg detail_log "$detail_log" \ + '. + { + timestamp: $timestamp, + engine: $engine, + tp: $tp, + dp: $dp, + mark: $mark, + isl: $isl, + dsl: $dsl, + concurrency: $concurrency, + num_prompts: $num_prompts, + warmup_requests: $warmup_requests, + attempt: $attempt, + gain_pct: $gain_pct, + plateau_streak: $plateau_streak, + error_type: "", + raw_file: $raw_file, + detail_log: $detail_log + }' "$metrics_file" >> "$ADAPTIVE_POINTS_JSONL" +} + +adaptive_append_failed_point() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local dsl="$5" + local concurrency="$6" + local num_prompts="$7" + local attempt="$8" + local status="$9" + local error_type="${10}" + local raw_file="${11}" + local detail_log="${12}" + + jq -nc \ + --arg timestamp "$(adaptive_timestamp)" \ + --arg engine "$ENGINE" \ + --argjson tp "$tp" \ + --argjson dp "$dp" \ + --arg mark "$mark" \ + --argjson isl "$isl" \ + --argjson dsl "$dsl" \ + --argjson concurrency "$concurrency" \ + --argjson num_prompts "$num_prompts" \ + --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ + --argjson attempt "$attempt" \ + --arg status "$status" \ + --arg error_type "$error_type" \ + --arg raw_file "$raw_file" \ + --arg detail_log "$detail_log" \ + '{ + timestamp: $timestamp, + engine: $engine, + tp: $tp, + dp: $dp, + mark: $mark, + isl: $isl, + dsl: $dsl, + concurrency: $concurrency, + num_prompts: $num_prompts, + warmup_requests: $warmup_requests, + attempt: $attempt, + status: $status, + completed: 0, + failed: $num_prompts, + error_type: $error_type, + validation_errors: [], + gain_pct: null, + plateau_streak: 0, + raw_file: $raw_file, + detail_log: $detail_log + }' >> "$ADAPTIVE_POINTS_JSONL" +} + +adaptive_append_shape_summary() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local dsl="$5" + local status="$6" + local stop_reason="$7" + local tested_points="$8" + local max_successful_concurrency="$9" + local saturation_concurrency="${10:-}" + local stop_probe_concurrency="${11:-}" + local best_tps_concurrency="${12:-}" + local best_total_tps="${13:-}" + local last_total_tps="${14:-}" + local saturation_json stop_probe_json best_c_json best_tps_json last_tps_json + saturation_json="$(adaptive_json_number_or_null "$saturation_concurrency")" + stop_probe_json="$(adaptive_json_number_or_null "$stop_probe_concurrency")" + best_c_json="$(adaptive_json_number_or_null "$best_tps_concurrency")" + best_tps_json="$(adaptive_json_number_or_null "$best_total_tps")" + last_tps_json="$(adaptive_json_number_or_null "$last_total_tps")" + + jq -nc \ + --arg timestamp "$(adaptive_timestamp)" \ + --arg engine "$ENGINE" \ + --argjson tp "$tp" \ + --argjson dp "$dp" \ + --arg mark "$mark" \ + --argjson isl "$isl" \ + --argjson dsl "$dsl" \ + --arg status "$status" \ + --arg stop_reason "$stop_reason" \ + --argjson tested_points "$tested_points" \ + --argjson search_cap "$SEARCH_MAX_CONCURRENCY" \ + --argjson max_successful_concurrency "$max_successful_concurrency" \ + --argjson saturation_concurrency "$saturation_json" \ + --argjson stop_probe_concurrency "$stop_probe_json" \ + --argjson best_tps_concurrency "$best_c_json" \ + --argjson best_total_tps "$best_tps_json" \ + --argjson last_total_tps "$last_tps_json" \ + '{ + timestamp: $timestamp, + engine: $engine, + tp: $tp, + dp: $dp, + mark: $mark, + isl: $isl, + dsl: $dsl, + status: $status, + stop_reason: $stop_reason, + tested_points: $tested_points, + search_cap: $search_cap, + max_successful_concurrency: $max_successful_concurrency, + saturation_concurrency: $saturation_concurrency, + stop_probe_concurrency: $stop_probe_concurrency, + best_tps_concurrency: $best_tps_concurrency, + best_total_tps: $best_total_tps, + last_total_tps: $last_total_tps + }' >> "$ADAPTIVE_SHAPES_JSONL" +} + +adaptive_run_point() { + local tp="$1" + local dp="$2" + local isl="$3" + local dsl="$4" + local concurrency="$5" + local num_prompts="$6" + local config_root="$7" + local attempt bench_rc parse_rc gpu_pid + + POINT_RAW_FILE="" + POINT_DETAIL_LOG="" + POINT_METRICS_FILE="" + POINT_ATTEMPT=0 + POINT_ERROR_TYPE="" + + for (( attempt = 1; attempt <= MAX_POINT_RETRIES + 1; attempt++ )); do + POINT_ATTEMPT="$attempt" + POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${dsl}_a${attempt}.jsonl" + POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${dsl}_a${attempt}.log" + POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${dsl}_a${attempt}.json" + local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${dsl}_a${attempt}.csv" + + log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}" + gpu_pid="$(adaptive_start_gpu_monitor "$gpu_csv")" + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" bash -c \ + 'engine_run_bench "$@"' _ \ + "$isl" "$dsl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \ + > "$POINT_DETAIL_LOG" 2>&1 || bench_rc=$? + adaptive_stop_gpu_monitor "$gpu_pid" + + if (( bench_rc == 0 )); then + parse_rc=0 + "$PYTHON" "$ADAPTIVE_HELPER" parse-result \ + --input "$POINT_RAW_FILE" \ + --output "$POINT_METRICS_FILE" \ + --expected-prompts "$num_prompts" \ + --isl "$isl" \ + --dsl "$dsl" \ + --input-tolerance-pct "$INPUT_LENGTH_TOLERANCE_PCT" \ + --output-tolerance-pct "$OUTPUT_LENGTH_TOLERANCE_PCT" \ + >> "$POINT_DETAIL_LOG" 2>&1 || parse_rc=$? + if (( parse_rc == 0 )); then + return 0 + fi + POINT_ERROR_TYPE="workload validation failed" + return 21 + fi + + if engine_detect_oom "$POINT_DETAIL_LOG" "$tp" "$dp"; then + POINT_ERROR_TYPE="detected out of memory" + return 20 + fi + + POINT_ERROR_TYPE="benchmark rc=${bench_rc}" + if (( attempt <= MAX_POINT_RETRIES )); then + log "probe failed without OOM; restarting server before retry" + if ! adaptive_restart_server "$tp" "$dp"; then + POINT_ERROR_TYPE="benchmark failed and server restart failed" + return 22 + fi + continue + fi + return 22 + done +} + +adaptive_run_shape() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local dsl="$5" + local config_root="$6" + local concurrency="$SEARCH_START_CONCURRENCY" + local previous_tps="" + local current_tps="" + local best_tps="" + local best_concurrency="" + local max_successful_concurrency=0 + local last_total_tps="" + local plateau_streak=0 + local plateau_start_concurrency="" + local saturation_concurrency="" + local stop_probe_concurrency="" + local stop_reason="SEARCH_CAP_REACHED" + local shape_status="COMPLETED" + local tested_points=0 + local gain_pct="" + local meaningful_gain=1 + local point_rc=0 + + ADAPTIVE_RESTART_NEEDED=0 + ADAPTIVE_FATAL_WORKLOAD=0 + + while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do + local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER)) + point_rc=0 + adaptive_run_point "$tp" "$dp" "$isl" "$dsl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$? + tested_points=$((tested_points + 1)) + stop_probe_concurrency="$concurrency" + + if (( point_rc == 0 )); then + current_tps="$(jq -r '.total_tps' "$POINT_METRICS_FILE")" + gain_pct="" + meaningful_gain=1 + if [[ -n "$previous_tps" ]]; then + IFS=$'\t' read -r gain_pct meaningful_gain < <( + "$PYTHON" "$ADAPTIVE_HELPER" gain \ + --previous "$previous_tps" \ + --current "$current_tps" \ + --threshold-pct "$TPS_MIN_GAIN_PCT" + ) + if (( meaningful_gain == 1 )); then + plateau_streak=0 + plateau_start_concurrency="" + else + plateau_streak=$((plateau_streak + 1)) + if [[ -z "$plateau_start_concurrency" ]]; then + plateau_start_concurrency="$concurrency" + fi + fi + fi + + if [[ -z "$best_tps" ]] || awk -v current="$current_tps" -v best="$best_tps" 'BEGIN { exit !(current > best) }'; then + best_tps="$current_tps" + best_concurrency="$concurrency" + fi + max_successful_concurrency="$concurrency" + last_total_tps="$current_tps" + adaptive_append_point_from_metrics \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$dsl" \ + "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ + "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + + log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}" + if (( plateau_streak >= PLATEAU_PATIENCE )); then + saturation_concurrency="$plateau_start_concurrency" + stop_reason="TPS_PLATEAU" + break + fi + previous_tps="$current_tps" + elif (( point_rc == 20 )); then + adaptive_append_failed_point \ + "$tp" "$dp" "$mark" "$isl" "$dsl" "$concurrency" "$num_prompts" \ + "$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + shape_status="OOM_BOUNDARY" + stop_reason="OOM" + ADAPTIVE_RESTART_NEEDED=1 + break + elif (( point_rc == 21 )); then + adaptive_append_point_from_metrics \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$dsl" \ + "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "" 0 \ + "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + shape_status="INVALID_WORKLOAD" + stop_reason="INVALID_WORKLOAD" + ADAPTIVE_FATAL_WORKLOAD=1 + break + else + adaptive_append_failed_point \ + "$tp" "$dp" "$mark" "$isl" "$dsl" "$concurrency" "$num_prompts" \ + "$POINT_ATTEMPT" "FAILED" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + shape_status="FAILED" + stop_reason="BENCHMARK_FAILED" + ADAPTIVE_RESTART_NEEDED=1 + break + fi + + if (( concurrency == SEARCH_MAX_CONCURRENCY )); then + stop_reason="SEARCH_CAP_REACHED" + break + fi + local next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then + next_concurrency="$SEARCH_MAX_CONCURRENCY" + fi + concurrency="$next_concurrency" + done + + adaptive_append_shape_summary \ + "$tp" "$dp" "$mark" "$isl" "$dsl" "$shape_status" "$stop_reason" \ + "$tested_points" "$max_successful_concurrency" "$saturation_concurrency" \ + "$stop_probe_concurrency" "$best_concurrency" "$best_tps" "$last_total_tps" + + log "shape done tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}" +} + +adaptive_matches_filter() { + local value="$1" + local filter="${2:-}" + [[ -z "$filter" || " $filter " == *" $value "* ]] +} + +adaptive_print_sequence() { + local concurrency="$SEARCH_START_CONCURRENCY" + local -a sequence=() + while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do + sequence+=("$concurrency") + if (( concurrency == SEARCH_MAX_CONCURRENCY )); then + break + fi + local next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then + next_concurrency="$SEARCH_MAX_CONCURRENCY" + fi + concurrency="$next_concurrency" + done + printf '%s ' "${sequence[@]}" +} + +adaptive_cleanup_active_server() { + if [[ -n "${ADAPTIVE_ACTIVE_TP:-}" && -n "${ADAPTIVE_ACTIVE_DP:-}" ]]; then + engine_stop_server "$ADAPTIVE_ACTIVE_TP" "$ADAPTIVE_ACTIVE_DP" || true + ADAPTIVE_ACTIVE_TP="" + ADAPTIVE_ACTIVE_DP="" + fi +} + +adaptive_main() { + RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}" + RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" + MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" + MATRIX_MODE="${MATRIX_MODE:-Y}" + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}" + ADAPTIVE_LOG_DIR="${ADAPTIVE_RUN_ROOT}/logs" + ADAPTIVE_POINTS_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_points.jsonl" + ADAPTIVE_SHAPES_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" + local shapes_tsv="${ADAPTIVE_RUN_ROOT}/shapes.tsv" + mkdir -p "$ADAPTIVE_LOG_DIR" + : > "$ADAPTIVE_POINTS_JSONL" + : > "$ADAPTIVE_SHAPES_JSONL" + log_init "${ADAPTIVE_LOG_DIR}/orchestrator.log" + + adaptive_validate_config + "$PYTHON" "$ADAPTIVE_HELPER" shapes \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$shapes_tsv" + + local tokenize_prompt_json=true + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + local required_dataset_rows=$((SEARCH_MAX_CONCURRENCY * NUM_PROMPTS_MULTIPLIER)) + local dataset_capacity=0 + local dataset_error="" + if [[ ! -f "$DATASET_PATH" ]]; then + dataset_error="DATASET_PATH does not exist: ${DATASET_PATH}" + elif ! dataset_capacity="$( + "$PYTHON" "$ADAPTIVE_HELPER" dataset-capacity --path "$DATASET_PATH" 2>/dev/null + )"; then + dataset_error="DATASET_PATH is not a valid ShareGPT JSON file: ${DATASET_PATH}" + fi + if [[ -n "$dataset_error" ]]; then + if [[ "$DRY_RUN" == "1" ]]; then + log "WARNING: ${dataset_error}" + else + log "ERROR: ${dataset_error}" + return 1 + fi + fi + if [[ -z "$dataset_error" ]] && (( dataset_capacity < required_dataset_rows )); then + if [[ "$DRY_RUN" == "1" ]]; then + log "WARNING: DATASET_PATH has ${dataset_capacity} usable conversations, but a real run may request ${required_dataset_rows}." + else + log "ERROR: DATASET_PATH has ${dataset_capacity} usable conversations, but adaptive search may request ${required_dataset_rows}. Set DATASET_PATH to a larger ShareGPT file or explicitly use BENCH_DATASET_NAME=random-ids." + return 1 + fi + fi + tokenize_prompt_json=false + elif [[ "$BENCH_DATASET_NAME" != random* ]]; then + log "ERROR: BENCH_DATASET_NAME must be random or start with random" + return 1 + fi + + log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}" + log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}" + log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}" + + jq -n \ + --arg experiment "$EXPERIMENT_NAME" \ + --arg engine "$ENGINE" \ + --arg run_id "$RUN_ID" \ + --arg model "$MODEL_PATH" \ + --arg hardware "$HARDWARE" \ + --arg matrix "$MATRIX_FILE" \ + --arg dataset "$BENCH_DATASET_NAME" \ + --argjson tokenize_prompt "$tokenize_prompt_json" \ + --argjson random_range_ratio "$RANDOM_RANGE_RATIO" \ + --argjson search_start "$SEARCH_START_CONCURRENCY" \ + --argjson search_cap "$SEARCH_MAX_CONCURRENCY" \ + --argjson search_multiplier "$SEARCH_MULTIPLIER" \ + --argjson prompts_multiplier "$NUM_PROMPTS_MULTIPLIER" \ + --argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \ + --argjson plateau_patience "$PLATEAU_PATIENCE" \ + --argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \ + '{ + experiment: $experiment, + engine: $engine, + run_id: $run_id, + model: $model, + hardware: $hardware, + matrix: $matrix, + dataset: $dataset, + tokenize_prompt: $tokenize_prompt, + random_range_ratio: $random_range_ratio, + search: { + start_concurrency: $search_start, + max_concurrency: $search_cap, + multiplier: $search_multiplier, + num_prompts_multiplier: $prompts_multiplier, + min_tps_gain_pct: $min_gain_pct, + plateau_patience: $plateau_patience, + warmup_max_requests: $warmup_max_requests + } + }' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" + + ADAPTIVE_ACTIVE_TP="" + ADAPTIVE_ACTIVE_DP="" + trap adaptive_cleanup_active_server EXIT + trap 'adaptive_cleanup_active_server; exit 130' INT + trap 'adaptive_cleanup_active_server; exit 143' TERM + + local abort_all=0 + local cfg tp dp config_root server_args shape_count mark isl dsl + for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + if ! adaptive_matches_filter "$tp" "$TP_LIST"; then + continue + fi + config_root="${ADAPTIVE_RUN_ROOT}/tp${tp}_dp${dp}" + mkdir -p "${config_root}/raw_outputs" "${config_root}/metrics" \ + "${config_root}/logs" "${config_root}/gpu_logs" + server_args="$(engine_build_server_args "$tp" "$dp")" + printf '%s\n' "$server_args" > "${config_root}/server_cmd.txt" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN server tp=${tp} dp=${dp}: ${server_args}" + shape_count=0 + while IFS=$'\t' read -r mark isl dsl; do + [[ "$mark" == "mark" ]] && continue + adaptive_matches_filter "$isl" "$ISL_LIST" || continue + adaptive_matches_filter "$dsl" "$DSL_LIST" || continue + if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then + break + fi + shape_count=$((shape_count + 1)) + log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} C=[$(adaptive_print_sequence)]" + done < "$shapes_tsv" + continue + fi + + ADAPTIVE_ACTIVE_TP="$tp" + ADAPTIVE_ACTIVE_DP="$dp" + if ! engine_start_server "$tp" "$dp"; then + log "ERROR: failed to start ${ENGINE} tp=${tp} dp=${dp}; skipping config" + adaptive_cleanup_active_server + continue + fi + adaptive_warmup "${ADAPTIVE_LOG_DIR}/${ENGINE}_tp${tp}_dp${dp}.warmup.log" || { + log "ERROR: warmup failed for tp=${tp} dp=${dp}; skipping config" + adaptive_cleanup_active_server + continue + } + + shape_count=0 + while IFS=$'\t' read -r mark isl dsl; do + [[ "$mark" == "mark" ]] && continue + adaptive_matches_filter "$isl" "$ISL_LIST" || continue + adaptive_matches_filter "$dsl" "$DSL_LIST" || continue + if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then + log "GRID_LIMIT=${GRID_LIMIT} reached for tp=${tp} dp=${dp}" + break + fi + shape_count=$((shape_count + 1)) + log "===== shape START tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} mark=${mark} =====" + adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$dsl" "$config_root" + + if (( ADAPTIVE_FATAL_WORKLOAD == 1 )); then + log "ERROR: invalid synthetic workload; aborting remaining searches" + abort_all=1 + break + fi + if (( ADAPTIVE_RESTART_NEEDED == 1 )); then + if ! adaptive_restart_server "$tp" "$dp"; then + log "ERROR: server restart failed; skipping remaining shapes for tp=${tp} dp=${dp}" + break + fi + fi + done < "$shapes_tsv" + + adaptive_cleanup_active_server + if (( abort_all == 1 )); then + break + fi + done + + "$PYTHON" "$ADAPTIVE_HELPER" summarize \ + --points "$ADAPTIVE_POINTS_JSONL" \ + --shapes "$ADAPTIVE_SHAPES_JSONL" \ + --output-dir "$ADAPTIVE_RUN_ROOT" \ + >> "${ADAPTIVE_LOG_DIR}/summarize.log" 2>&1 + log "adaptive results saved to ${ADAPTIVE_RUN_ROOT}" + + trap - EXIT INT TERM + adaptive_cleanup_active_server + if (( abort_all == 1 )); then + return 2 + fi +} diff --git a/scripts/common/adaptive_concurrency.py b/scripts/common/adaptive_concurrency.py new file mode 100755 index 0000000..d7a517b --- /dev/null +++ b/scripts/common/adaptive_concurrency.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Helpers for adaptive-concurrency serving benchmarks.""" + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path +from typing import Any, Iterable + + +POINT_FIELDS = [ + "timestamp", + "engine", + "tp", + "dp", + "mark", + "isl", + "dsl", + "concurrency", + "num_prompts", + "warmup_requests", + "attempt", + "status", + "completed", + "failed", + "duration_s", + "request_tps", + "input_tps", + "output_tps", + "total_tps", + "mean_input_tokens", + "mean_output_tokens", + "ttft_p50_ms", + "ttft_p95_ms", + "ttft_p99_ms", + "tpot_p50_ms", + "tpot_p95_ms", + "tpot_p99_ms", + "e2e_p50_ms", + "e2e_p95_ms", + "e2e_p99_ms", + "itl_p50_ms", + "itl_p95_ms", + "itl_p99_ms", + "gain_pct", + "plateau_streak", + "error_type", + "validation_errors", + "raw_file", + "detail_log", +] + + +SHAPE_FIELDS = [ + "timestamp", + "engine", + "tp", + "dp", + "mark", + "isl", + "dsl", + "status", + "stop_reason", + "tested_points", + "search_cap", + "max_successful_concurrency", + "saturation_concurrency", + "stop_probe_concurrency", + "best_tps_concurrency", + "best_total_tps", + "last_total_tps", +] + + +def read_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + if not path.exists(): + return rows + with path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + rows.append(value) + return rows + + +def last_json_object(path: Path) -> dict[str, Any]: + rows = read_jsonl(path) + if not rows: + raise ValueError(f"no valid JSON object in {path}") + return rows[-1] + + +def percentile_value(data: dict[str, Any], prefix: str, percentile: str) -> float: + if percentile == "p50": + key = f"median_{prefix}_ms" + else: + key = f"{percentile}_{prefix}_ms" + return float(data.get(key, 0.0) or 0.0) + + +def command_shapes(args: argparse.Namespace) -> int: + with args.matrix.open("r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode or data.get("mode", "Y") + print("mark\tinput_len\toutput_len") + for isl_text in sorted(data["matrix"], key=int): + dsl_map = data["matrix"][isl_text] + for dsl_text in sorted(dsl_map, key=int): + mark = dsl_map[dsl_text] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + if mode != "all" and mark == "N": + continue + print(f"{mark}\t{int(isl_text)}\t{int(dsl_text)}") + return 0 + + +def command_dataset_capacity(args: argparse.Namespace) -> int: + try: + with args.path.open("r", encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"invalid dataset {args.path}: {exc}") + if not isinstance(data, list): + raise SystemExit(f"dataset must be a JSON list: {args.path}") + capacity = 0 + for row in data: + if not isinstance(row, dict): + continue + conversations = row.get("conversations", row.get("conversation", [])) + if isinstance(conversations, list) and len(conversations) >= 2: + capacity += 1 + print(capacity) + return 0 + + +def command_parse_result(args: argparse.Namespace) -> int: + validation_errors: list[str] = [] + try: + data = last_json_object(args.input) + except (OSError, ValueError) as exc: + data = {} + validation_errors.append(str(exc)) + + completed = int(data.get("completed", 0) or 0) + input_lens = [int(v) for v in data.get("input_lens", []) if v is not None] + output_lens = [int(v) for v in data.get("output_lens", []) if v is not None] + errors = [str(v) for v in data.get("errors", []) if str(v)] + mean_input = sum(input_lens) / len(input_lens) if input_lens else 0.0 + mean_output = sum(output_lens) / len(output_lens) if output_lens else 0.0 + + if completed != args.expected_prompts: + validation_errors.append( + f"completed={completed}, expected={args.expected_prompts}" + ) + if errors: + validation_errors.append(f"request_errors={len(errors)}") + + input_low = args.isl * (1.0 - args.input_tolerance_pct / 100.0) + input_high = args.isl * (1.0 + args.input_tolerance_pct / 100.0) + if not input_low <= mean_input <= input_high: + validation_errors.append( + f"mean_input_tokens={mean_input:.2f}, expected_range={input_low:.2f}-{input_high:.2f}" + ) + + output_low = args.dsl * (1.0 - args.output_tolerance_pct / 100.0) + output_high = args.dsl * (1.0 + args.output_tolerance_pct / 100.0) + if not output_low <= mean_output <= output_high: + validation_errors.append( + f"mean_output_tokens={mean_output:.2f}, expected_range={output_low:.2f}-{output_high:.2f}" + ) + + status = "COMPLETED" if not validation_errors else "INVALID_WORKLOAD" + result = { + "status": status, + "completed": completed, + "failed": max(args.expected_prompts - completed, len(errors)), + "duration_s": float(data.get("duration", 0.0) or 0.0), + "request_tps": float(data.get("request_throughput", 0.0) or 0.0), + "input_tps": float(data.get("input_throughput", 0.0) or 0.0), + "output_tps": float(data.get("output_throughput", 0.0) or 0.0), + "total_tps": float(data.get("total_throughput", 0.0) or 0.0), + "mean_input_tokens": mean_input, + "mean_output_tokens": mean_output, + "ttft_p50_ms": percentile_value(data, "ttft", "p50"), + "ttft_p95_ms": percentile_value(data, "ttft", "p95"), + "ttft_p99_ms": percentile_value(data, "ttft", "p99"), + "tpot_p50_ms": percentile_value(data, "tpot", "p50"), + "tpot_p95_ms": percentile_value(data, "tpot", "p95"), + "tpot_p99_ms": percentile_value(data, "tpot", "p99"), + "e2e_p50_ms": percentile_value(data, "e2e_latency", "p50"), + "e2e_p95_ms": percentile_value(data, "e2e_latency", "p95"), + "e2e_p99_ms": percentile_value(data, "e2e_latency", "p99"), + "itl_p50_ms": percentile_value(data, "itl", "p50"), + "itl_p95_ms": percentile_value(data, "itl", "p95"), + "itl_p99_ms": percentile_value(data, "itl", "p99"), + "validation_errors": validation_errors, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as f: + json.dump(result, f, indent=2, ensure_ascii=False) + f.write("\n") + print(status) + return 0 if status == "COMPLETED" else 2 + + +def command_gain(args: argparse.Namespace) -> int: + if args.previous <= 0: + gain_pct = float("inf") + meaningful = True + else: + gain_pct = (args.current - args.previous) / args.previous * 100.0 + meaningful = gain_pct >= args.threshold_pct + gain_text = "inf" if gain_pct == float("inf") else f"{gain_pct:.6f}" + print(f"{gain_text}\t{1 if meaningful else 0}") + return 0 + + +def write_csv(path: Path, rows: Iterable[dict[str, Any]], fields: list[str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + for row in rows: + output = dict(row) + if isinstance(output.get("validation_errors"), list): + output["validation_errors"] = "; ".join(output["validation_errors"]) + writer.writerow(output) + + +def command_summarize(args: argparse.Namespace) -> int: + points = read_jsonl(args.points) + shapes = read_jsonl(args.shapes) + args.output_dir.mkdir(parents=True, exist_ok=True) + write_csv(args.output_dir / "adaptive_points.csv", points, POINT_FIELDS) + write_csv(args.output_dir / "adaptive_summary.csv", shapes, SHAPE_FIELDS) + + summary_jsonl = args.output_dir / "adaptive_summary.jsonl" + with summary_jsonl.open("w", encoding="utf-8") as f: + for row in shapes: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + report = args.output_dir / "adaptive_summary.md" + with report.open("w", encoding="utf-8") as f: + f.write("# Adaptive concurrency search summary\n\n") + f.write( + "| Engine | TP | DP | ISL | DSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C |\n" + ) + f.write("|---|---:|---:|---:|---:|---|---:|---:|---:|---:|\n") + for row in shapes: + f.write( + "| {engine} | {tp} | {dp} | {isl} | {dsl} | {stop_reason} | " + "{saturation_concurrency} | {best_tps_concurrency} | {best_total_tps} | " + "{max_successful_concurrency} |\n".format(**{k: row.get(k, "") for k in SHAPE_FIELDS}) + ) + f.write("\n") + f.write( + "`Saturation C` is the first point in the final low-gain streak. " + "`Best TPS C` is the tested point with the highest observed Total TPS.\n" + ) + print(f"wrote summaries under {args.output_dir}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + shapes = subparsers.add_parser("shapes") + shapes.add_argument("--matrix", type=Path, required=True) + shapes.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None) + shapes.set_defaults(func=command_shapes) + + dataset_capacity = subparsers.add_parser("dataset-capacity") + dataset_capacity.add_argument("--path", type=Path, required=True) + dataset_capacity.set_defaults(func=command_dataset_capacity) + + parse_result = subparsers.add_parser("parse-result") + parse_result.add_argument("--input", type=Path, required=True) + parse_result.add_argument("--output", type=Path, required=True) + parse_result.add_argument("--expected-prompts", type=int, required=True) + parse_result.add_argument("--isl", type=int, required=True) + parse_result.add_argument("--dsl", type=int, required=True) + parse_result.add_argument("--input-tolerance-pct", type=float, default=5.0) + parse_result.add_argument("--output-tolerance-pct", type=float, default=10.0) + parse_result.set_defaults(func=command_parse_result) + + gain = subparsers.add_parser("gain") + gain.add_argument("--previous", type=float, required=True) + gain.add_argument("--current", type=float, required=True) + gain.add_argument("--threshold-pct", type=float, required=True) + gain.set_defaults(func=command_gain) + + summarize = subparsers.add_parser("summarize") + summarize.add_argument("--points", type=Path, required=True) + summarize.add_argument("--shapes", type=Path, required=True) + summarize.add_argument("--output-dir", type=Path, required=True) + summarize.set_defaults(func=command_summarize) + return parser + + +def main() -> int: + args = build_parser().parse_args() + return int(args.func(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) From cf7a12932e26f38eb00527ff1dcbbe0ff523b97d Mon Sep 17 00:00:00 2001 From: Root User Date: Sat, 11 Jul 2026 13:04:35 +0000 Subject: [PATCH 41/53] bench: add adaptive run heartbeat monitor --- scripts/common/adaptive_heartbeat.sh | 104 +++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 scripts/common/adaptive_heartbeat.sh diff --git a/scripts/common/adaptive_heartbeat.sh b/scripts/common/adaptive_heartbeat.sh new file mode 100755 index 0000000..b5366be --- /dev/null +++ b/scripts/common/adaptive_heartbeat.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Lightweight heartbeat monitor for a long-running adaptive benchmark. +set -uo pipefail + +if (( $# < 2 )); then + echo "Usage: $0 RUN_ROOT MAIN_TMUX [ENGINE_PORT] [SERVICE_NAME_PATTERN]" >&2 + exit 2 +fi + +RUN_ROOT="$1" +MAIN_TMUX="$2" +ENGINE_PORT="${3:-}" +SERVICE_NAME_PATTERN="${4:-}" +INTERVAL_S="${HEARTBEAT_INTERVAL_S:-60}" +LOG_FILE="${HEARTBEAT_LOG_FILE:-${RUN_ROOT}/heartbeat.log}" + +if ! [[ "$INTERVAL_S" =~ ^[1-9][0-9]*$ ]]; then + echo "HEARTBEAT_INTERVAL_S must be a positive integer" >&2 + exit 2 +fi + +mkdir -p "$(dirname "$LOG_FILE")" + +line_count() { + local path="$1" + if [[ -f "$path" ]]; then + wc -l < "$path" | tr -d ' ' + else + printf '0' + fi +} + +gpu_summary() { + nvidia-smi \ + --query-gpu=index,memory.used,utilization.gpu \ + --format=csv,noheader,nounits 2>/dev/null | \ + awk -F',' ' + { + for (i = 1; i <= NF; i++) gsub(/^[[:space:]]+|[[:space:]]+$/, "", $i) + printf "%s%s:%sMiB/%s%%", (NR == 1 ? "" : ";"), $1, $2, $3 + } + END { if (NR == 0) printf "unavailable" } + ' +} + +heartbeat_once() { + local timestamp tmux_state health_state container_names container_state + local points shapes last_line gpu + + timestamp="$(date --iso-8601=seconds)" + if tmux has-session -t "$MAIN_TMUX" 2>/dev/null; then + tmux_state="up" + else + tmux_state="down" + fi + + health_state="n/a" + if [[ -n "$ENGINE_PORT" ]]; then + if curl --fail --silent --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1; then + health_state="ok" + else + health_state="down" + fi + fi + + container_names="" + if [[ -n "$SERVICE_NAME_PATTERN" ]]; then + container_names="$( + docker ps --filter "name=${SERVICE_NAME_PATTERN}" --format '{{.Names}}' 2>/dev/null | \ + paste -sd, - + )" + fi + if [[ -n "$container_names" ]]; then + container_state="$container_names" + else + container_state="none" + fi + + points="$(line_count "${RUN_ROOT}/adaptive_points.jsonl")" + shapes="$(line_count "${RUN_ROOT}/adaptive_shapes.jsonl")" + last_line="$( + tail -n 1 "${RUN_ROOT}/logs/orchestrator.log" 2>/dev/null | \ + tr '\t\r\n' ' ' | cut -c1-320 + )" + gpu="$(gpu_summary)" + + printf '[%s] benchmark_tmux=%s health=%s service=%s points=%s shapes=%s gpu=%s last="%s"\n' \ + "$timestamp" "$tmux_state" "$health_state" "$container_state" \ + "$points" "$shapes" "$gpu" "$last_line" | tee -a "$LOG_FILE" + + [[ "$tmux_state" == "up" ]] +} + +printf '[%s] heartbeat_started interval_s=%s main_tmux=%s run_root=%s\n' \ + "$(date --iso-8601=seconds)" "$INTERVAL_S" "$MAIN_TMUX" "$RUN_ROOT" | \ + tee -a "$LOG_FILE" + +while heartbeat_once; do + sleep "$INTERVAL_S" +done + +printf '[%s] heartbeat_stopped reason=benchmark_tmux_down\n' \ + "$(date --iso-8601=seconds)" | tee -a "$LOG_FILE" From 8ebe9632697912df346e8014f3dd04012087af98 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 13 Jul 2026 02:18:34 +0000 Subject: [PATCH 42/53] Add dsv4_pro6000_sglang_tp_dp_matrix experiment for RTX 6000D - New experiment directory with adaptive concurrency benchmark for SGLang - Platform config for 8x NVIDIA RTX 6000D 84GB - Adapted from dsv4_h200_sglang_tp_dp_matrix with updated paths: - Model path: /data/hf_models/DeepSeek-V4-Flash - Venv path: /root/.miniconda3/envs/sglang - Dataset path: /data/yy/sskj/datasets/ --- .../adaptive_config.env | 49 +++++ .../config.env | 71 +++++++ .../run_adaptive_concurrency.sh | 181 ++++++++++++++++++ .../run_sglang_in_container.sh | 93 +++++++++ .../start_sglang_container.sh | 70 +++++++ .../start_sglang_docker.sh | 99 ++++++++++ .../start_sglang_dp.sh | 86 +++++++++ platforms/nvidia_rtx6000d.env | 23 +++ 8 files changed, 672 insertions(+) create mode 100644 experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env create mode 100644 experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh create mode 100644 platforms/nvidia_rtx6000d.env diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..4085397 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, DSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/DSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +DSL_LIST="${DSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env new file mode 100644 index 0000000..c963f5c --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# TP×DP matrix experiment for DeepSeek-V4-Flash on RTX 6000D (8 GPUs) using SGLang. +# Tests SGLang with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_pro6000_sglang_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +SGLANG_PORT="${SGLANG_PORT:-30031}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# SGLang server settings +CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" +MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" + +# Deployment switch. 0 = native sglang venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline nodes. +DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..90f7827 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local dsl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$dsl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh new file mode 100755 index 0000000..69a08b0 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Run SGLang server inside the already-running benchmark container. +# Usage: run_sglang_in_container.sh +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" + +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_in_container_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Build server args. +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server inside container (TP=${TP}, DP=${DP}) ===" +echo "Container: $NAME" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Kill any existing sglang process inside container first. +docker exec "$NAME" pkill -9 -f "sglang serve" 2>/dev/null || true +sleep 3 + +# Start sglang serve inside the container in background. +# We use nohup so it survives after docker exec returns. +docker exec -d \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$NAME" \ + bash -c "nohup sglang ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &" + +# Wait for the server process to appear inside container. +sleep 2 +SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang serve" | head -n 1 || true) +if [[ -z "$SERVER_PID" ]]; then + echo "ERROR: sglang server process not found inside container" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -50 || true + exit 1 +fi + +echo "Server PID inside container: $SERVER_PID" +echo "$SERVER_PID" > "$PID_FILE" + +echo "Waiting for health on port ${PORT}..." +for i in $(seq 1 360); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + # Check if server process is still alive. + if ! docker exec "$NAME" kill -0 "$SERVER_PID" 2>/dev/null; then + echo "ERROR: SGLang server exited early" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true + exit 1 + fi + echo "Waiting... ($i/360)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 360 retries (30 mins)" +docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true +exit 1 diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh new file mode 100755 index 0000000..2cf469c --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Start a long-running Docker container for SGLang benchmarking. +# The container stays alive (sleep infinity) so we can docker exec into it +# to run different TP×DP configurations without losing DeepGEMM JIT cache. +# +# Usage: start_sglang_container.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_container.pid" + +# Persistent cache directory on host for DeepGEMM JIT kernels. +CACHE_DIR="${RUNTIME_BASE}/cache" +mkdir -p "${CACHE_DIR}/deep_gemm" "${CACHE_DIR}/tvm-ffi" + +# Clean up any stale container. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_container_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +echo "=== Starting SGLang benchmark container ===" +echo "Image: $IMAGE" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Cache dir: $CACHE_DIR" +echo "Log: $LOG" + +# Start a long-running container. +# We override the entrypoint to sleep infinity so the container stays alive. +docker run -d \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint /bin/bash \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}/deep_gemm:/root/.cache/deep_gemm" \ + -v "${CACHE_DIR}/tvm-ffi:/root/.cache/tvm-ffi" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + -c "sleep infinity" \ + > "$LOG" 2>&1 + +# Wait a moment for container to be ready. +sleep 2 + +if ! docker ps --filter "name=$NAME" --format '{{.Names}}' | grep -q "$NAME"; then + echo "ERROR: Container failed to start" + tail -50 "$LOG" + exit 1 +fi + +# Record the container ID for later use. +docker inspect -f '{{.Id}}' "$NAME" > "$PID_FILE" + +echo "Container $NAME is running" +echo "Log: $LOG" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh new file mode 100755 index 0000000..5753e46 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Start SGLang server in Docker for a given TPxDP configuration. +# Usage: start_sglang_docker.sh +# +# Uses the lmsysorg/sglang image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint sglang \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh new file mode 100755 index 0000000..230bf88 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the +# local VENV_CLIENT environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_CLIENT}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + sglang serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${SGLANG_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/platforms/nvidia_rtx6000d.env b/platforms/nvidia_rtx6000d.env new file mode 100644 index 0000000..5f720e9 --- /dev/null +++ b/platforms/nvidia_rtx6000d.env @@ -0,0 +1,23 @@ +# Platform configuration for NVIDIA RTX 6000D (Pro 6000D) +# Source this file via scripts/common/platform.sh + +CHIP="nvidia_rtx6000d" +ACCELERATOR="NVIDIA RTX 6000D" +HARDWARE="8x NVIDIA RTX 6000D 84GB" +ENGINE="sglang" + +# Device selection +DEVICE_SELECT_ENV="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7" +CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" + +# Default serving port and model root on the host +DEFAULT_PORT="30004" +MODEL_ROOT="/data/hf_models" + +# Virtual environments on the host (used by native scripts). +# Override these if your machine uses different paths. +VENV_SGLANG="${VENV_SGLANG:-/root/.miniconda3/envs/sglang}" +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Default server start script for the legacy benchmark grid. +SERVER_START_SCRIPT="${SERVER_START_SCRIPT:-${ROOT_DIR}/scripts/start_dsv4_dspark_8card.sh}" From 302d1a9822d051a1ada1b0a7205d114a60222948 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 13 Jul 2026 02:28:11 +0000 Subject: [PATCH 43/53] Rename DSL to OSL (Output Sequence Length) in dsv4_pro6000 experiment DSL -> OSL to better reflect the meaning: Output Sequence Length. --- .../adaptive_config.env | 10 +++++----- .../run_adaptive_concurrency.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env index 4085397..1eda1f0 100644 --- a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env @@ -1,6 +1,6 @@ # Adaptive concurrency search settings. # -# For each fixed (TP, DP, ISL, DSL), probe: +# For each fixed (TP, DP, ISL, OSL), probe: # C = start, start * multiplier, ... up to max # and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for # PLATEAU_PATIENCE consecutive points. @@ -21,7 +21,7 @@ PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" # random-ids to use generated token IDs without a ShareGPT seed dataset. BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed -# ISL/DSL points; lower values intentionally benchmark a length distribution. +# ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" # Before each measured point, warm up with the same concurrency so lazy kernel # compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no @@ -39,11 +39,11 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" # Optional space-separated filters, useful for smoke tests: -# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" TP_LIST="${TP_LIST:-}" ISL_LIST="${ISL_LIST:-}" -DSL_LIST="${DSL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" DRY_RUN="${DRY_RUN:-0}" -# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 90f7827..0979a46 100755 --- a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -130,7 +130,7 @@ engine_detect_oom() { engine_run_bench() { local isl="$1" - local dsl="$2" + local osl="$2" local concurrency="$3" local num_prompts="$4" local output_file="$5" @@ -142,7 +142,7 @@ engine_run_bench() { --port "$ENGINE_PORT" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" - --random-output-len "$dsl" + --random-output-len "$osl" --random-range-ratio "$RANDOM_RANGE_RATIO" --num-prompts "$num_prompts" --max-concurrency "$concurrency" From 372ff08eea3c36d8a63b5bc3c32bac1e67b42b8f Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 13 Jul 2026 02:46:15 +0000 Subject: [PATCH 44/53] Add master orchestrator script for sequential benchmark execution - Implemented a bash script to run six benchmark experiments sequentially. - Added features for fault tolerance, out-of-memory recovery, and checkpoint management. - Included detailed logging for monitoring experiment progress and results. - Integrated GPU memory cleanup checks before each experiment. - Provided functionality to resume experiments from checkpoints. --- .../bench_client.py | 53 +- .../config.env | 184 +- .../results/20260710-091522/results.json | 47 - .../results/20260710-091700/results.json | 47 - .../results.json | 6 +- .../README.md | 273 - .../bench_client.py | 617 -- .../config.env | 84 - .../parse_results.py | 206 - .../results/20260710-091141/results.json | 47 - .../results/20260710-091522/results.json | 47 - .../results/20260710-091700/results.json | 47 - .../run_bench.sh | 277 - .../start_server.sh | 109 - .../bench_client.py | 53 +- .../config.env | 184 +- .../results/20260711-133908/report.md | 129 + .../results/20260711-133908/results.json | 6429 +++++++++++++++++ .../README.md | 126 - .../bench_client.py | 617 -- .../config.env | 84 - .../parse_results.py | 206 - .../run_bench.sh | 278 - .../start_server.sh | 107 - .../bench_client.py | 53 +- .../config.env | 185 +- .../results/20260711-193756/report.md | 129 + .../results/20260711-193756/results.json | 6428 ++++++++++++++++ .../run_bench.sh | 1 - .../README.md | 103 - .../bench_client.py | 617 -- .../config.env | 83 - .../parse_results.py | 206 - .../run_bench.sh | 272 - .../start_server.sh | 106 - 35 files changed, 13671 insertions(+), 4769 deletions(-) delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json rename experiments/dsv4_h200_vllm_tp2_custom_bench/results/{20260710-091141 => 20260710-104930}/results.json (91%) delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py delete mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py delete mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py index d8edc36..9deaa5b 100755 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py @@ -228,16 +228,22 @@ class LoadBalancer: # Token helpers # --------------------------------------------------------------------------- -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: +import uuid +def make_prompt(token_len: int) -> str: + prefix = str(uuid.uuid4()) + + words = [ + "the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "benchmark", "performance", "latency" + ] + + tokens = [prefix] + + while len(tokens) < token_len: tokens.extend(words) - return " ".join(tokens[:needed]) + + return " ".join(tokens[:token_len]) # --------------------------------------------------------------------------- @@ -440,7 +446,36 @@ async def run_benchmark( return await send_request_with_retry( session, host, lb, model, idx, prompt, output_len, temperature ) + print("Starting warmup...") + warmup_per_service = 10 + warmup_requests = warmup_per_service * len(ports) + warmup_prompts = [ + make_prompt(input_len) + for _ in range(warmup_requests) + ] + + warmup_tasks = [ + send_request_with_retry( + session=session, + host=host, + lb=lb, + model=model, + request_id=-1, + prompt=p, + max_tokens=output_len, + temperature=temperature, + ) + for p in warmup_prompts + ] + + await asyncio.gather(*warmup_tasks) + + print("Warmup finished.") + + # 给 CUDA / Scheduler 一点时间稳定 + await asyncio.sleep(2) + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] results = await asyncio.gather(*tasks) await health_checker.stop() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env index 2b2d56c..edee0e7 100644 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env @@ -34,50 +34,146 @@ LB_STRATEGY="round_robin" # For multi-service we include low-concurrency scenarios to show how # requests spread across 4 services. SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + + "1 1024 128 20" + "1 1024 256 20" + "1 1024 512 20" + "1 1024 1024 20" + "1 1024 2048 20" + "1 1024 4096 20" + + "8 1024 128 160" + "8 1024 256 160" + "8 1024 512 160" + "8 1024 1024 160" + "8 1024 2048 160" + "8 1024 4096 160" + + "32 1024 128 640" + "32 1024 256 640" + "32 1024 512 640" + "32 1024 1024 640" + "32 1024 2048 640" + "32 1024 4096 640" + + "128 1024 128 2560" + "128 1024 256 2560" + "128 1024 512 2560" + "128 1024 1024 2560" + "128 1024 2048 2560" + "128 1024 4096 2560" + + "1 2048 128 20" + "1 2048 256 20" + "1 2048 512 20" + "1 2048 1024 20" + "1 2048 2048 20" + "1 2048 4096 20" + + "8 2048 128 160" + "8 2048 256 160" + "8 2048 512 160" + "8 2048 1024 160" + "8 2048 2048 160" + "8 2048 4096 160" + + "32 2048 128 640" + "32 2048 256 640" + "32 2048 512 640" + "32 2048 1024 640" + "32 2048 2048 640" + "32 2048 4096 640" + + "128 2048 128 2560" + "128 2048 256 2560" + "128 2048 512 2560" + "128 2048 1024 2560" + "128 2048 2048 2560" + "128 2048 4096 2560" + + "1 4096 128 20" + "1 4096 256 20" + "1 4096 512 20" + "1 4096 1024 20" + "1 4096 2048 20" + "1 4096 4096 20" + + "8 4096 128 160" + "8 4096 256 160" + "8 4096 512 160" + "8 4096 1024 160" + "8 4096 2048 160" + "8 4096 4096 160" + + "32 4096 128 640" + "32 4096 256 640" + "32 4096 512 640" + "32 4096 1024 640" + "32 4096 2048 640" + "32 4096 4096 640" + + "128 4096 128 2560" + "128 4096 256 2560" + "128 4096 512 2560" + "128 4096 1024 2560" + "128 4096 2048 2560" + "128 4096 4096 2560" + + "1 16384 128 20" + "1 16384 256 20" + "1 16384 512 20" + "1 16384 1024 20" + "1 16384 2048 20" + + "8 16384 128 160" + "8 16384 256 160" + "8 16384 512 160" + "8 16384 1024 160" + "8 16384 2048 160" + + "32 16384 128 640" + "32 16384 256 640" + "32 16384 512 640" + "32 16384 1024 640" + "32 16384 2048 640" + + "128 16384 128 2560" + "128 16384 256 2560" + "128 16384 512 2560" + "128 16384 1024 2560" + "128 16384 2048 2560" + + + "1 65536 128 20" + "1 65536 256 20" + "1 65536 512 20" + "1 65536 1024 20" + + "8 65536 128 160" + "8 65536 256 160" + "8 65536 512 160" + "8 65536 1024 160" + "32 65536 1024 160" + + "1 131072 128 20" + "1 131072 256 20" + "1 131072 512 20" + + "8 131072 512 160" + + "1 262144 256 20" + + "8 262144 128 160" + + "1 524288 128 20" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json deleted file mode 100644 index 3196c06..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091522", - "timestamp": "2026-07-10T09:15:22+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json deleted file mode 100644 index 16c1892..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091700", - "timestamp": "2026-07-10T09:17:00+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json similarity index 91% rename from experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json rename to experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json index 64dd782..5c17b81 100644 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json @@ -1,8 +1,8 @@ { "metadata": { "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091141", - "timestamp": "2026-07-10T09:11:41+00:00", + "run_id": "20260710-104930", + "timestamp": "2026-07-10T10:49:30+00:00", "model": "/data/models/DeepSeek-V4-Flash", "backend": "vllm", "engine": "vllm", @@ -11,7 +11,7 @@ "chip": "nvidia_h200", "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", + "git_commit": "def1355", "git_dirty": "dirty", "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" }, diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md deleted file mode 100644 index dae2993..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md +++ /dev/null @@ -1,273 +0,0 @@ -# dsv4_h200_vllm_tp2_custom_bench_no_fp8 - -> **Note:** This variant does **NOT** use `--kv-cache-dtype fp8`. - -## 目的 - -当 vLLM 使用 **TP=2**（Tensor Parallel = 2）部署时，单张 H200 上可以同时运行 **4 个独立服务**（8 卡 / 2 = 4 服务），每个服务独占 2 张 GPU： - -| 服务 | 端口 | GPU 对 | -|------|------|--------| -| 1 | 30005 | 0, 1 | -| 2 | 30006 | 2, 3 | -| 3 | 30007 | 4, 5 | -| 4 | 30008 | 6, 7 | - -此时 `sglang.bench_serving` 存在以下问题： - -1. **低并发下无法测出真实吞吐**：并发数不大时，请求可能只落在一个服务上，其他 3 个服务空闲，bench 报告的吞吐只是单服务的 1/4。 -2. **TTFT 测量不够精确**：`sglang.bench_serving` 的计时粒度较粗，对于低延迟场景（TP=2 下 TTFT 可能 < 100ms）误差较大。 -3. **不支持多服务负载均衡**：只能测一个端口，无法反映整个 8 卡集群性能。 - -因此本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API，通过 `aiohttp` 并发发送请求，并记录每个请求的精确时间戳。同时通过 **负载均衡** 将请求均匀分发到 4 个服务上。 - -## 与 sglang.bench_serving 的区别 - -| 特性 | sglang.bench_serving | bench_client.py (本实验) | -|------|----------------------|--------------------------| -| 协议 | 内部协议 | OpenAI API (`/v1/chat/completions`) | -| 并发控制 | 进程级 | `asyncio.Semaphore` | -| TTFT 测量 | 粗略 | 精确到 `time.perf_counter()` | -| TPOT 测量 | 基于总时间估算 | 基于首 token 后每个 token 的间隔 | -| 多服务支持 | ❌ 单端口 | ✅ 轮询/随机分发到多个端口 | -| 低并发精度 | 一般 | 高（适合 TP=2 场景） | -| 集群吞吐 | 只能测单服务 | 可测 4 服务总吞吐 | - -## 文件说明 - -| 文件 | 说明 | -|------|------| -| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡 | -| `config.env` | 实验配置（场景、模型路径、端口列表、LB 策略等） | -| `run_bench.sh` | 编排脚本：启动 4 个服务 → 运行压测 → 解析结果 | -| `start_server.sh` | 同时启动 4 个 vLLM TP=2 服务（每服务 2 GPU） | -| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | - -## 快速运行 - -```bash -# 完整流程（自动启动 4 个服务、压测、生成报告） -bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh - -# 跳过服务管理（已有 4 个服务在运行） -SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh - -# 单独运行压测客户端（多服务，带健康检查） -python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ - --lb-strategy round_robin \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file /tmp/test.jsonl - -# 单独运行压测客户端（单服务，向后兼容） -python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ - --output-file /tmp/test_single.jsonl - -# 解析已有结果 -python3 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py \ - experiments/dsv4_h200_vllm_tp2_custom_bench/results/ -``` - -## 场景设计 - -本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： - -| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | -|------|----------|----------|--------|------| -| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | -| 4 | 512 | 256 | 80 | 低并发，4 服务各 1 req | -| 8 | 512 | 256 | 160 | 中低并发 | -| 16 | 512 | 256 | 320 | 中等并发 | -| 32 | 512 | 256 | 640 | 中高并发 | -| 64 | 512 | 256 | 1280 | 高并发 | -| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | -| 1 | 2048 | 512 | 20 | 长输入基线 | -| 8 | 2048 | 512 | 160 | 长输入中并发 | -| 32 | 2048 | 512 | 640 | 长输入高并发 | -| 128 | 2048 | 512 | 2560 | 长输入极限并发 | -| 1 | 4096 | 1024 | 20 | 超长输入基线 | -| 8 | 4096 | 1024 | 160 | 超长输入中并发 | -| 32 | 4096 | 1024 | 640 | 超长输入高并发 | -| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | - -## 健康检查与故障自动跳过 - -`bench_client.py` 内置了 **后台健康检查 + 请求级重试** 机制，确保某个服务挂了不会导致整个 benchmark 失败。 - -### 健康检查机制 - -| 参数 | 默认值 | 说明 | -|------|--------|------| -| `--health-check-interval` | 5.0 | 每 N 秒检查一次所有服务 | -| `--health-max-failures` | 2 | 连续失败 N 次才标记为不健康 | -| `--health-recovery-interval` | 10.0 | 不健康服务 N 秒后自动重试恢复 | -| `--request-max-retries` | 2 | 单个请求失败最多重试 N 次 | - -### 故障处理流程 - -``` -请求 → 负载均衡选择端口 → 发送请求 - ↓ -成功 → 记录结果 - ↓ -失败 → 换另一个端口重试（最多 --request-max-retries 次） - ↓ -全部失败 → 记录失败，继续下一个请求 -``` - -后台同时运行健康检查： -- 每 5 秒对所有端口执行 `/health` 探测 -- 连续 2 次失败的服务被标记为 **unhealthy**，不再接收新请求 -- 10 秒后自动尝试恢复，如果恢复成功则重新加入负载均衡池 - -### 输出中的故障信息 - -汇总 JSON 包含以下字段： - -```json -{ - "completed": 638, - "failed": 2, - "retried": 5, - "healthy_ports_at_end": [30005, 30006, 30007], - "per_port_stats": { - "30005": {"count": 160, "failed": 0, ...}, - "30006": {"count": 159, "failed": 1, ...}, - "30007": {"count": 160, "failed": 0, ...}, - "30008": {"count": 159, "failed": 1, ...} - } -} -``` - -- `failed`: 最终失败的请求数（重试后仍失败） -- `retried`: 至少重试过一次的请求数 -- `healthy_ports_at_end`: 压测结束时仍健康的服务端口 -- `per_port_stats[].failed`: 每个端口的失败数 - -### 场景间健康检查 - -`run_bench.sh` 在每个场景结束后会打印一次服务健康状态： - -``` -[2026-07-10T08:45:00+00:00] service health check: healthy=[30005 30006 30007 30008] unhealthy=[] -[2026-07-10T08:45:30+00:00] service health check: healthy=[30005 30006 30007] unhealthy=[30008] -[2026-07-10T08:45:30+00:00] WARNING: 1 service(s) unhealthy: 30008 -``` - -这样即使某个服务在压测中途 OOM 崩溃，其他服务仍能继续工作，benchmark 不会中断。 - -### 自定义故障参数 - -```bash -python3 bench_client.py \ - --ports 30005,30006,30007,30008 \ - --health-check-interval 3 \ - --health-max-failures 1 \ - --health-recovery-interval 5 \ - --request-max-retries 3 \ - ... -``` - -- 更敏感：间隔 3 秒、1 次失败即标记不健康、5 秒恢复 -- 更容错：单个请求最多重试 3 次 - -## 负载均衡策略 - -`bench_client.py` 支持三种负载均衡策略： - -| 策略 | 说明 | 适用场景 | -|------|------|----------| -| `round_robin` | 请求按顺序轮询分配到各端口 | **默认**，最公平，推荐 | -| `random` | 随机选择一个端口 | 简单，分布可能不均 | -| `least_conn` | 选择当前连接数最少的端口 | 需要追踪 in-flight，当前为 random 占位 | - -通过 `--lb-strategy` 参数选择： - -```bash -python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,30008 ... -``` - -## 输出格式 - -### JSONL 原始数据 (`raw_outputs/vllm_*.jsonl`) - -每行一个 JSON 对象，包含请求分发到的目标端口： - -```json -{ - "request_id": 0, - "input_tokens": 512, - "output_tokens": 256, - "first_token_time_ms": 45.23, - "e2e_latency_ms": 5234.56, - "inter_token_latencies": [18.5, 19.2, ...], - "tpot_ms": 19.8, - "mean_itl_ms": 19.5, - "success": true, - "error": "", - "target_port": 30005 -} -``` - -最后一行是汇总数据，包含 **各端口统计**： - -```json -{ - "completed": 640, - "failed": 0, - "duration": 12.34, - "request_throughput": 51.88, - "mean_ttft_ms": 48.5, - "p95_ttft_ms": 62.3, - "per_port_stats": { - "30005": {"count": 160, "mean_ttft_ms": 47.2, "p95_ttft_ms": 58.1, "mean_e2e_ms": 5200.0}, - "30006": {"count": 160, "mean_ttft_ms": 49.1, "p95_ttft_ms": 64.3, "mean_e2e_ms": 5250.0}, - "30007": {"count": 160, "mean_ttft_ms": 48.8, "p95_ttft_ms": 61.5, "mean_e2e_ms": 5230.0}, - "30008": {"count": 160, "mean_ttft_ms": 48.9, "p95_ttft_ms": 65.2, "mean_e2e_ms": 5270.0} - }, - "lb_strategy": "round_robin", - "num_services": 4 -} -``` - -### 结构化结果 (`results.json`) - -遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。 - -### 人类可读报告 (`report.md`) - -Markdown 表格，包含所有场景的 TTFT/TPOT/E2E 均值与分位数，以及 SLO 达标状态。 - -## 单独启动/停止 4 个服务 - -```bash -# 启动 4 个服务 -bash experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh - -# 停止（run_bench.sh 会自动停止，但也可以手动） -for port in 30005 30006 30007 30008; do - pid_file="experiments/dsv4_h200_vllm_tp2_custom_bench/server_port${port}.pid" - [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true - rm -f "$pid_file" -done -pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true -``` - -## 注意事项 - -- `bench_client.py` 优先使用 `aiohttp`，如果未安装则自动回退到 `urllib`（但会丢失 streaming 和精确 TTFT/ITL 测量）。 -- 默认 `temperature=0.0` 以保证输出长度稳定。 -- 输入 token 数通过空格分隔的单词数估算，实际 tokenizer 可能略有偏差。 -- 4 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 -- 每个服务占用约 2 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 -- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 -- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py deleted file mode 100755 index d8edc36..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py +++ /dev/null @@ -1,617 +0,0 @@ -#!/usr/bin/env python3 -"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. - -Sends concurrent requests to multiple inference servers and records per-request -latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. - -Designed for TP=2 on 8-GPU setups where 4 independent services run on -(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via -round-robin or random load balancing. Unhealthy services are automatically -skipped with periodic retry. - -Usage: - # Single service (backward compatible) - python3 bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --output-file results.jsonl - - # Multi-service (4 services on ports 30005-30008) - python3 bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --lb-strategy round_robin \ - --output-file results.jsonl - -The output JSONL contains one object per request with raw timing data. -A final summary line ("completed" field) aggregates all requests. -""" - -import argparse -import asyncio -import json -import random -import sys -import time -import urllib.request -from dataclasses import asdict, dataclass, field -from typing import Any - -# Try aiohttp first; fall back to urllib for stdlib-only environments. -try: - import aiohttp - HAS_AIOHTTP = True -except ImportError: - HAS_AIOHTTP = False - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - -@dataclass -class RequestResult: - request_id: int - input_tokens: int = 0 - output_tokens: int = 0 - first_token_time_ms: float = 0.0 # TTFT - e2e_latency_ms: float = 0.0 # total wall time - inter_token_latencies: list[float] = field(default_factory=list) - success: bool = False - error: str = "" - target_port: int = 0 # which service handled this request - retry_count: int = 0 # how many times this request was retried - - @property - def tpot_ms(self) -> float: - """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" - if self.output_tokens <= 1: - return 0.0 - return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) - - @property - def mean_itl_ms(self) -> float: - if not self.inter_token_latencies: - return 0.0 - return sum(self.inter_token_latencies) / len(self.inter_token_latencies) - - def to_dict(self) -> dict[str, Any]: - d = asdict(self) - d["tpot_ms"] = self.tpot_ms - d["mean_itl_ms"] = self.mean_itl_ms - return d - - -# --------------------------------------------------------------------------- -# Health checker -# --------------------------------------------------------------------------- - -class HealthChecker: - """Periodically check service health and maintain a healthy-port list.""" - - def __init__( - self, - host: str, - ports: list[int], - check_interval: float = 5.0, - max_failures: int = 2, - recovery_interval: float = 10.0, - ): - self.host = host - self.all_ports = ports - self.check_interval = check_interval - self.max_failures = max_failures - self.recovery_interval = recovery_interval - - # port -> consecutive failure count - self._failures: dict[int, int] = {p: 0 for p in ports} - # port -> last healthy timestamp - self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} - self._lock = asyncio.Lock() - self._task: asyncio.Task | None = None - - async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: - url = f"http://{self.host}:{port}/health" - try: - if HAS_AIOHTTP and session is not None: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: - return resp.status == 200 - else: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - return resp.status == 200 - except Exception: - return False - - async def _check_all(self, session: aiohttp.ClientSession | None) -> None: - checks = [self._check_one(p, session) for p in self.all_ports] - results = await asyncio.gather(*checks, return_exceptions=True) - now = time.monotonic() - async with self._lock: - for port, ok in zip(self.all_ports, results): - if isinstance(ok, Exception): - ok = False - if ok: - self._failures[port] = 0 - self._last_healthy[port] = now - else: - self._failures[port] += 1 - - async def start(self, session: aiohttp.ClientSession | None) -> None: - async def _loop() -> None: - while True: - await asyncio.sleep(self.check_interval) - await self._check_all(session) - self._task = asyncio.create_task(_loop()) - - async def stop(self) -> None: - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - @property - async def healthy_ports(self) -> list[int]: - now = time.monotonic() - async with self._lock: - healthy = [] - for p in self.all_ports: - if self._failures[p] < self.max_failures: - healthy.append(p) - elif now - self._last_healthy[p] > self.recovery_interval: - # Give it another chance after recovery_interval. - self._failures[p] = max(0, self._failures[p] - 1) - healthy.append(p) - return healthy - - async def is_healthy(self, port: int) -> bool: - ports = await self.healthy_ports - return port in ports - - -# --------------------------------------------------------------------------- -# Load balancer with failover -# --------------------------------------------------------------------------- - -class LoadBalancer: - """Distribute requests across healthy backend ports with auto-failover.""" - - def __init__( - self, - ports: list[int], - health_checker: HealthChecker, - strategy: str = "round_robin", - ): - self.all_ports = ports - self.health_checker = health_checker - self.strategy = strategy - self._idx = 0 - self._lock = asyncio.Lock() - - def _next_round_robin(self, healthy: list[int]) -> int: - if not healthy: - return self.all_ports[self._idx % len(self.all_ports)] - for _ in range(len(self.all_ports)): - port = self.all_ports[self._idx % len(self.all_ports)] - self._idx += 1 - if port in healthy: - return port - # Fallback: all unhealthy, pick first healthy anyway. - return healthy[0] if healthy else self.all_ports[0] - - def _next_random(self, healthy: list[int]) -> int: - if healthy: - return random.choice(healthy) - return random.choice(self.all_ports) - - async def next_port(self) -> int: - healthy = await self.health_checker.healthy_ports - async with self._lock: - if self.strategy == "round_robin": - return self._next_round_robin(healthy) - elif self.strategy in ("random", "least_conn"): - return self._next_random(healthy) - else: - return self._next_round_robin(healthy) - - @property - def port_count(self) -> int: - return len(self.all_ports) - - -# --------------------------------------------------------------------------- -# Token helpers -# --------------------------------------------------------------------------- - -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: - tokens.extend(words) - return " ".join(tokens[:needed]) - - -# --------------------------------------------------------------------------- -# aiohttp-based async request (preferred) -# --------------------------------------------------------------------------- - -async def send_request_aiohttp( - session: aiohttp.ClientSession, - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - } - headers = {"Content-Type": "application/json"} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - first_token_received = False - last_token_time = 0.0 - - try: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status != 200: - text = await resp.text() - result.error = f"HTTP {resp.status}: {text[:200]}" - return result - - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or not line.startswith("data: "): - continue - data_str = line[len("data: "):] - if data_str == "[DONE]": - break - try: - chunk = json.loads(data_str) - except json.JSONDecodeError: - continue - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - now = time.perf_counter() - if not first_token_received: - result.first_token_time_ms = (now - start_time) * 1000 - first_token_received = True - else: - result.inter_token_latencies.append((now - last_token_time) * 1000) - last_token_time = now - result.output_tokens += 1 - - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.input_tokens = len(prompt.split()) # approximate - result.success = True - - except asyncio.TimeoutError: - result.error = "timeout" - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# urllib-based synchronous request (fallback, no aiohttp) -# --------------------------------------------------------------------------- - -def send_request_urllib( - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": False, - } - data = json.dumps(payload).encode("utf-8") - headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - - try: - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=600) as resp: - body = json.loads(resp.read().decode("utf-8")) - - choices = body.get("choices", []) - if not choices: - result.error = "no choices in response" - return result - - content = choices[0].get("message", {}).get("content", "") - result.output_tokens = len(content.split()) if content else 0 - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.first_token_time_ms = result.e2e_latency_ms - result.input_tokens = len(prompt.split()) - result.success = True - - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# Unified send_request wrapper with retry & failover -# --------------------------------------------------------------------------- - -async def send_request_with_retry( - session: aiohttp.ClientSession | None, - host: str, - lb: LoadBalancer, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, - max_retries: int = 2, -) -> RequestResult: - """Send a request, retrying on failure with a different port each time.""" - result = RequestResult(request_id=request_id) - for attempt in range(max_retries + 1): - port = await lb.next_port() - if HAS_AIOHTTP and session is not None: - result = await send_request_aiohttp( - session, host, port, model, request_id, prompt, max_tokens, temperature - ) - else: - loop = asyncio.get_event_loop() - result = await loop.run_in_executor( - None, send_request_urllib, - host, port, model, request_id, prompt, max_tokens, temperature - ) - result.retry_count = attempt - if result.success: - return result - # If failed, try another port on next iteration (unless last attempt). - if attempt < max_retries: - await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish - return result - - -# --------------------------------------------------------------------------- -# Benchmark orchestration -# --------------------------------------------------------------------------- - -async def run_benchmark( - host: str, - ports: list[int], - lb: LoadBalancer, - health_checker: HealthChecker, - model: str, - concurrency: int, - input_len: int, - output_len: int, - num_prompts: int, - temperature: float, - output_file: str, -) -> dict[str, Any]: - """Run the benchmark and write a JSONL file.""" - prompts = [make_prompt(input_len) for _ in range(num_prompts)] - results: list[RequestResult] = [] - semaphore = asyncio.Semaphore(concurrency) - - start = time.perf_counter() - - if HAS_AIOHTTP: - connector = aiohttp.TCPConnector(limit=concurrency * 2) - timeout = aiohttp.ClientTimeout(total=600, sock_read=300) - async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: - await health_checker.start(session) - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - session, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - else: - await health_checker.start(None) - from concurrent.futures import ThreadPoolExecutor - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - None, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - - duration = time.perf_counter() - start - - # Write JSONL - with open(output_file, "w", encoding="utf-8") as f: - for r in results: - f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") - - # Compute summary - successful = [r for r in results if r.success] - failed = len(results) - len(successful) - retried = [r for r in results if r.retry_count > 0] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - ttfts = [r.first_token_time_ms for r in successful] - tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] - e2es = [r.e2e_latency_ms for r in successful] - itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] - - total_input_tokens = sum(r.input_tokens for r in successful) - total_output_tokens = sum(r.output_tokens for r in successful) - - # Per-port breakdown - per_port_stats: dict[int, dict[str, Any]] = {} - for port in ports: - port_results = [r for r in successful if r.target_port == port] - port_failed = [r for r in results if r.target_port == port and not r.success] - if port_results or port_failed: - port_ttfts = [r.first_token_time_ms for r in port_results] - port_e2es = [r.e2e_latency_ms for r in port_results] - per_port_stats[port] = { - "count": len(port_results), - "failed": len(port_failed), - "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, - "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, - "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, - } - - # Health-check stats - healthy_at_end = await health_checker.healthy_ports - - summary = { - "completed": len(successful), - "failed": failed, - "retried": len(retried), - "duration": duration, - "request_throughput": len(successful) / duration if duration > 0 else 0.0, - "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, - "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, - "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, - "total_input_tokens": total_input_tokens, - "total_output_tokens": total_output_tokens, - "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, - "median_ttft_ms": percentile(ttfts, 50), - "p90_ttft_ms": percentile(ttfts, 90), - "p95_ttft_ms": percentile(ttfts, 95), - "p99_ttft_ms": percentile(ttfts, 99), - "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, - "median_tpot_ms": percentile(tpots, 50), - "p90_tpot_ms": percentile(tpots, 90), - "p95_tpot_ms": percentile(tpots, 95), - "p99_tpot_ms": percentile(tpots, 99), - "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, - "median_e2e_latency_ms": percentile(e2es, 50), - "p90_e2e_latency_ms": percentile(e2es, 90), - "p95_e2e_latency_ms": percentile(e2es, 95), - "p99_e2e_latency_ms": percentile(e2es, 99), - "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, - "median_itl_ms": percentile(itls, 50), - "p90_itl_ms": percentile(itls, 90), - "p95_itl_ms": percentile(itls, 95), - "p99_itl_ms": percentile(itls, 99), - "per_port_stats": per_port_stats, - "healthy_ports_at_end": healthy_at_end, - "lb_strategy": lb.strategy, - "num_services": lb.port_count, - } - - # Append summary as the last line - with open(output_file, "a", encoding="utf-8") as f: - f.write(json.dumps(summary, ensure_ascii=False) + "\n") - - return summary - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30005, - help="Single service port (backward compatible)") - parser.add_argument("--ports", default="", - help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") - parser.add_argument("--model", default="deepseek-v4-flash") - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--input-len", type=int, default=512) - parser.add_argument("--output-len", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=10) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--lb-strategy", default="round_robin", - choices=["round_robin", "random", "least_conn"], - help="Load balancing strategy across services") - parser.add_argument("--health-check-interval", type=float, default=5.0, - help="Seconds between health checks (default: 5)") - parser.add_argument("--health-max-failures", type=int, default=2, - help="Consecutive failures before marking a port unhealthy (default: 2)") - parser.add_argument("--health-recovery-interval", type=float, default=10.0, - help="Seconds before retrying an unhealthy port (default: 10)") - parser.add_argument("--request-max-retries", type=int, default=2, - help="Max retries per request on failure (default: 2)") - parser.add_argument("--output-file", required=True) - args = parser.parse_args() - - # Determine ports: --ports takes precedence, else fall back to --port. - if args.ports: - ports = [int(p.strip()) for p in args.ports.split(",")] - else: - ports = [args.port] - - health_checker = HealthChecker( - host=args.host, - ports=ports, - check_interval=args.health_check_interval, - max_failures=args.health_max_failures, - recovery_interval=args.health_recovery_interval, - ) - lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) - - summary = asyncio.run(run_benchmark( - host=args.host, - ports=ports, - lb=lb, - health_checker=health_checker, - model=args.model, - concurrency=args.concurrency, - input_len=args.input_len, - output_len=args.output_len, - num_prompts=args.num_prompts, - temperature=args.temperature, - output_file=args.output_file, - )) - - print(json.dumps(summary, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env deleted file mode 100644 index ddeb143..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env +++ /dev/null @@ -1,84 +0,0 @@ -# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench -# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. -# -# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: -# Service 1: port 30005, GPUs 0,1 -# Service 2: port 30006, GPUs 2,3 -# Service 3: port 30007, GPUs 4,5 -# Service 4: port 30008, GPUs 6,7 -# -# bench_client.py distributes requests across all 4 services via round-robin -# load balancing, giving accurate cluster-wide throughput even at low -# per-service concurrency. - -EXPERIMENT="dsv4_h200_vllm_tp2_custom_bench_no_fp8" -MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" - -# Primary port (backward compatible); multi-service ports are in PORTS. -PORT="30005" -PORTS="30005,30006,30007,30008" - -BACKEND="vllm" -ENGINE="vllm" - -# Native virtual environments on the host. -VENV_SERVER="/data/user1/yy/envs/vllm" -VENV_CLIENT="/data/user1/yy/envs/vllm" - -# Load balancing strategy: round_robin | random | least_conn -LB_STRATEGY="round_robin" - -# Benchmark scenarios: "concurrency input_len output_len num_prompts". -# For multi-service we include low-concurrency scenarios to show how -# requests spread across 4 services. -SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" -) - -# Server start script bundled with this experiment. -SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py deleted file mode 100755 index eea1a2e..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. - -Reads JSONL files produced by bench_client.py (one request per line, -last line is the summary) and generates: - - results.json (appended scenarios, following the standard schema) - - report.md (human-readable summary) - -Usage: - python3 parse_results.py -""" - -import json -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: - """Read the summary JSON (last line) and all per-request lines.""" - requests: list[dict] = [] - summary: dict | None = None - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - # The summary line has "completed" and "duration" keys. - if "completed" in obj and "duration" in obj: - summary = obj - else: - requests.append(obj) - return summary, requests - - -def compute_metrics(summary: dict, requests: list[dict]) -> dict: - completed = summary.get("completed", 0) - total = len(requests) - failed = total - completed if total > 0 else 0 - duration_s = summary.get("duration", 0.0) - - # Extract per-request metrics for percentiles. - ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] - tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] - e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] - itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": summary.get("request_throughput", 0.0), - "input_token_throughput": summary.get("input_throughput", 0.0), - "output_token_throughput": summary.get("output_throughput", 0.0), - "total_token_throughput": summary.get("total_throughput", 0.0), - "total_input_tokens": summary.get("total_input_tokens", 0), - "total_output_tokens": summary.get("total_output_tokens", 0), - "e2e_ms": { - "mean": summary.get("mean_e2e_latency_ms", 0.0), - "p50": percentile(e2es, 50), - "p90": percentile(e2es, 90), - "p95": percentile(e2es, 95), - "p99": percentile(e2es, 99), - }, - "ttft_ms": { - "mean": summary.get("mean_ttft_ms", 0.0), - "p50": percentile(ttfts, 50), - "p90": percentile(ttfts, 90), - "p95": percentile(ttfts, 95), - "p99": percentile(ttfts, 99), - }, - "tpot_ms": { - "mean": summary.get("mean_tpot_ms", 0.0), - "p50": percentile(tpots, 50), - "p90": percentile(tpots, 90), - "p95": percentile(tpots, 95), - "p99": percentile(tpots, 99), - }, - "itl_ms": { - "mean": summary.get("mean_itl_ms", 0.0), - "p50": percentile(itls, 50), - "p90": percentile(itls, 90), - "p95": percentile(itls, 95), - "p99": percentile(itls, 99), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: - ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms - tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") - f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") - f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") - f.write(f"- **Result root**: `{result_root}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - - summary, requests = parse_jsonl(jsonl_path) - if summary is None: - continue - - metrics = compute_metrics(summary, requests) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": "random", - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - for s in scenarios: - append_scenario(results_json, s) - - generate_report(result_root, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json deleted file mode 100644 index 64dd782..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091141", - "timestamp": "2026-07-10T09:11:41+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json deleted file mode 100644 index 3196c06..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091522", - "timestamp": "2026-07-10T09:15:22+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json deleted file mode 100644 index 16c1892..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091700", - "timestamp": "2026-07-10T09:17:00+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh deleted file mode 100755 index 6be99c8..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env bash -# Custom benchmark orchestrator for multi-service vLLM TP=2 on 8x H200. -# -# This runs 4 independent vLLM services (TP=2 each) on GPU pairs (0,1),(2,3),(4,5),(6,7). -# bench_client.py distributes requests across all 4 services via round-robin LB, -# giving accurate cluster-wide throughput even at low per-service concurrency. -# -# Usage: -# bash run_bench.sh -# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" - -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -ensure_result_root "$RESULT_ROOT" -log_init "${LOG_DIR}/orchestrator.log" - -log "experiment=${EXPERIMENT_NAME}" -log "run_id=${RUN_ID}" -log "result_root=${RESULT_ROOT}" -log "platform=${PLATFORM}" -log "chip=${CHIP}" -log "accelerator=${ACCELERATOR}" -log "engine=${ENGINE}" -log "hardware=${HARDWARE}" -log "model=${MODEL_PATH}" -log "services=${PORTS}" -log "lb_strategy=${LB_STRATEGY}" -log "server_start_script=${SERVER_START_SCRIPT}" - -# Write initial metadata for this run. -METADATA_JSON="${RESULT_ROOT}/results.json" -write_metadata_json \ - "$METADATA_JSON" \ - "$EXPERIMENT_NAME" \ - "$RUN_ID" \ - "$MODEL_PATH" \ - "$BACKEND" \ - "$ENGINE" \ - "$HARDWARE" \ - "$ACCELERATOR" \ - "$CHIP" \ - "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_SERVER" \ - "H200 4x vLLM TP=2 multi-service (no FP8 KV cache) benchmark using bench_client.py" - -# Update metadata with config. -"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' -import json, sys -path = sys.argv[1] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", "4 512 256 80", "8 512 256 160", - "16 512 256 320", "32 512 256 640", "64 512 256 1280", "128 512 256 2560", - "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", - "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" - ] -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY - -# --------------------------------------------------------------------------- -# Server helpers (multi-service) -# --------------------------------------------------------------------------- - -is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -are_all_services_healthy() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - for p in "${PORT_LIST[@]}"; do - if ! is_server_healthy "$p"; then - return 1 - fi - done - return 0 -} - -stop_all_servers() { - log "stopping all vllm services" - # Kill by PID files first. - for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do - [[ -f "$pid_file" ]] || continue - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "killing pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - done - # Fallback: kill any remaining vllm processes for this model. - pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true - sleep 2 -} - -start_all_servers() { - log "starting all vllm services with ${SERVER_START_SCRIPT}" - if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" - exit 1 - fi - bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - - if ! are_all_services_healthy "$PORTS"; then - log "error: not all vllm services became healthy" - exit 1 - fi - log "all vllm services are healthy" -} - -# --------------------------------------------------------------------------- -# Pre-benchmark health check with detailed reporting -# --------------------------------------------------------------------------- - -check_services_detailed() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy_count=0 - local unhealthy_ports="" - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - ((healthy_count++)) - else - unhealthy_ports="${unhealthy_ports}${p} " - fi - done - echo "$healthy_count" - if [[ -n "$unhealthy_ports" ]]; then - echo "unhealthy: ${unhealthy_ports}" >&2 - fi -} - -wait_for_all_services() { - local ports="$1" - local max_wait="${2:-240}" - local interval="${3:-5}" - local elapsed=0 - while (( elapsed < max_wait )); do - local healthy_count - healthy_count="$(check_services_detailed "$ports" | head -1)" - IFS=',' read -ra PORT_LIST <<< "$ports" - if (( healthy_count == ${#PORT_LIST[@]} )); then - return 0 - fi - log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" - sleep "$interval" - ((elapsed += interval)) - done - return 1 -} - -# --------------------------------------------------------------------------- -# Mid-benchmark health check (called between scenarios) -# --------------------------------------------------------------------------- - -check_and_report_services() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy=() - local unhealthy=() - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - healthy+=("$p") - else - unhealthy+=("$p") - fi - done - log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" - if [[ ${#unhealthy[@]} -gt 0 ]]; then - log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" - fi - return ${#unhealthy[@]} -} - -on_exit() { - local code=$? - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_all_servers - fi - exit "$code" -} -trap on_exit EXIT - -if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming services are already running" - if ! wait_for_all_services "$PORTS"; then - log "error: not all healthy services found on ports ${PORTS}" - exit 1 - fi -else - stop_all_servers - start_all_servers -fi - -log "===== BENCHMARK START =====" - -for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len num_prompts <<< "$scenario" - # Fallback for legacy 3-field scenarios. - if [[ -z "${num_prompts:-}" ]]; then - num_prompts="$NUM_PROMPTS" - fi - - output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" - - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ - --host 127.0.0.1 \ - --ports "$PORTS" \ - --model "$SERVED_MODEL_NAME" \ - --concurrency "$concurrency" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num-prompts "$num_prompts" \ - --lb-strategy "$LB_STRATEGY" \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file "$output_file" \ - > "$detail_log" 2>&1 || { - log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" - # After failure, check service health before continuing. - check_and_report_services "$PORTS" - continue - } - - log "finished scenario: output=${output_file}" - - # Between scenarios, report service health so we can spot degradation early. - check_and_report_services "$PORTS" || true -done - -log "===== BENCHMARK DONE =====" - -log "parsing results" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { - log "WARNING: parser failed; see ${LOG_DIR}/parse.log" -} - -log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh deleted file mode 100755 index 8555596..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash -# Start 4 independent vLLM TP=2 services on 8x H200. -# Each service runs on a pair of GPUs: (0,1), (2,3), (4,5), (6,7). -# This maximizes GPU utilization for low-concurrency workloads. -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -VENV="${VENV_SERVER}" -export PATH="$VENV/bin:$PATH" -VLLM="$VENV/bin/vllm" - -TP=2 -LOG_DIR="${SCRIPT_DIR}/logs" -mkdir -p "$LOG_DIR" - -# 4 services, each on 2 GPUs. -# Port and GPU pair mapping. -SERVICES=( - "30005:0,1" - "30006:2,3" - "30007:4,5" - "30008:6,7" -) - -# Kill any existing vLLM processes for this model. -pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true -sleep 2 - -# Remove old PID files. -rm -f "${SCRIPT_DIR}"/*.pid - -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - GPUS="${svc##*:}" - LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - - echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" - echo "Log: ${LOG}" - - CUDA_VISIBLE_DEVICES="${GPUS}" \ - nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --block-size 256 \ - --served-model-name "$SERVED_MODEL_NAME" \ - --port "$PORT" \ - > "$LOG" 2>&1 & - PID=$! - echo $PID > "$PID_FILE" - echo "PID: $PID" -done - -echo "" -echo "Waiting for all 4 services to become healthy..." - -MAX_WAIT=240 -all_healthy=0 -for i in $(seq 1 $MAX_WAIT); do - all_healthy=1 - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then - all_healthy=0 - break - fi - done - - if [[ "$all_healthy" -eq 1 ]]; then - echo "All 4 services are healthy!" - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - echo " - http://127.0.0.1:${PORT}" - done - exit 0 - fi - - # Check if any process died early. - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - if [[ -f "$PID_FILE" ]]; then - PID="$(cat "$PID_FILE")" - if ! kill -0 "$PID" 2>/dev/null; then - echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - tail -200 $LOG 2>/dev/null || true - exit 1 - fi - fi - done - - if (( i % 10 == 0 )); then - echo "Waiting... (${i}/${MAX_WAIT})" - fi - sleep 5 -done - -echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - echo "--- Last 50 lines of port ${PORT} ---" - tail -50 $LOG 2>/dev/null || true -done -exit 1 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py index d8edc36..9deaa5b 100755 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py @@ -228,16 +228,22 @@ class LoadBalancer: # Token helpers # --------------------------------------------------------------------------- -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: +import uuid +def make_prompt(token_len: int) -> str: + prefix = str(uuid.uuid4()) + + words = [ + "the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "benchmark", "performance", "latency" + ] + + tokens = [prefix] + + while len(tokens) < token_len: tokens.extend(words) - return " ".join(tokens[:needed]) + + return " ".join(tokens[:token_len]) # --------------------------------------------------------------------------- @@ -440,7 +446,36 @@ async def run_benchmark( return await send_request_with_retry( session, host, lb, model, idx, prompt, output_len, temperature ) + print("Starting warmup...") + warmup_per_service = 10 + warmup_requests = warmup_per_service * len(ports) + warmup_prompts = [ + make_prompt(input_len) + for _ in range(warmup_requests) + ] + + warmup_tasks = [ + send_request_with_retry( + session=session, + host=host, + lb=lb, + model=model, + request_id=-1, + prompt=p, + max_tokens=output_len, + temperature=temperature, + ) + for p in warmup_prompts + ] + + await asyncio.gather(*warmup_tasks) + + print("Warmup finished.") + + # 给 CUDA / Scheduler 一点时间稳定 + await asyncio.sleep(2) + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] results = await asyncio.gather(*tasks) await health_checker.stop() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env index 365ef84..881531e 100644 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env @@ -34,50 +34,146 @@ LB_STRATEGY="round_robin" # For multi-service we include low-concurrency scenarios to show how # requests spread across 2 services. SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + + "1 1024 128 20" + "1 1024 256 20" + "1 1024 512 20" + "1 1024 1024 20" + "1 1024 2048 20" + "1 1024 4096 20" + + "8 1024 128 160" + "8 1024 256 160" + "8 1024 512 160" + "8 1024 1024 160" + "8 1024 2048 160" + "8 1024 4096 160" + + "32 1024 128 640" + "32 1024 256 640" + "32 1024 512 640" + "32 1024 1024 640" + "32 1024 2048 640" + "32 1024 4096 640" + + "128 1024 128 2560" + "128 1024 256 2560" + "128 1024 512 2560" + "128 1024 1024 2560" + "128 1024 2048 2560" + "128 1024 4096 2560" + + "1 2048 128 20" + "1 2048 256 20" + "1 2048 512 20" + "1 2048 1024 20" + "1 2048 2048 20" + "1 2048 4096 20" + + "8 2048 128 160" + "8 2048 256 160" + "8 2048 512 160" + "8 2048 1024 160" + "8 2048 2048 160" + "8 2048 4096 160" + + "32 2048 128 640" + "32 2048 256 640" + "32 2048 512 640" + "32 2048 1024 640" + "32 2048 2048 640" + "32 2048 4096 640" + + "128 2048 128 2560" + "128 2048 256 2560" + "128 2048 512 2560" + "128 2048 1024 2560" + "128 2048 2048 2560" + "128 2048 4096 2560" + + "1 4096 128 20" + "1 4096 256 20" + "1 4096 512 20" + "1 4096 1024 20" + "1 4096 2048 20" + "1 4096 4096 20" + + "8 4096 128 160" + "8 4096 256 160" + "8 4096 512 160" + "8 4096 1024 160" + "8 4096 2048 160" + "8 4096 4096 160" + + "32 4096 128 640" + "32 4096 256 640" + "32 4096 512 640" + "32 4096 1024 640" + "32 4096 2048 640" + "32 4096 4096 640" + + "128 4096 128 2560" + "128 4096 256 2560" + "128 4096 512 2560" + "128 4096 1024 2560" + "128 4096 2048 2560" + "128 4096 4096 2560" + + "1 16384 128 20" + "1 16384 256 20" + "1 16384 512 20" + "1 16384 1024 20" + "1 16384 2048 20" + + "8 16384 128 160" + "8 16384 256 160" + "8 16384 512 160" + "8 16384 1024 160" + "8 16384 2048 160" + + "32 16384 128 640" + "32 16384 256 640" + "32 16384 512 640" + "32 16384 1024 640" + "32 16384 2048 640" + + "128 16384 128 2560" + "128 16384 256 2560" + "128 16384 512 2560" + "128 16384 1024 2560" + "128 16384 2048 2560" + + + "1 65536 128 20" + "1 65536 256 20" + "1 65536 512 20" + "1 65536 1024 20" + + "8 65536 128 160" + "8 65536 256 160" + "8 65536 512 160" + "8 65536 1024 160" + "32 65536 1024 160" + + "1 131072 128 20" + "1 131072 256 20" + "1 131072 512 20" + + "8 131072 512 160" + + "1 262144 256 20" + + "8 262144 128 160" + + "1 524288 128 20" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md new file mode 100644 index 0000000..4355fae --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md @@ -0,0 +1,129 @@ +# H200 vLLM TP=2 Custom Benchmark Report + +- **Client**: `bench_client.py` (async OpenAI API, per-request timing) +- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding) +- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i1024_o1024 | 128 | 1024 | 1024 | 172.39 | 2560 | 0 | 14.85 | 15206.17 | 3100.60 | 18306.76 | 267.16 | 478.80 | 2297.70 | 38.53 | 66.16 | 70.07 | 8265.00 | 13967.23 | 17536.46 | ✅ | +| c128_i1024_o128 | 128 | 1024 | 128 | 98.84 | 2560 | 0 | 25.90 | 26522.07 | 3314.91 | 29836.98 | 900.39 | 1590.87 | 2307.93 | 30.16 | 35.98 | 37.09 | 4730.89 | 5530.64 | 6122.15 | ✅ | +| c128_i1024_o2048 | 128 | 1024 | 2048 | 161.76 | 2560 | 0 | 15.83 | 16205.28 | 3297.74 | 19503.02 | 256.61 | 340.87 | 2315.99 | 36.07 | 39.27 | 40.30 | 7732.01 | 10616.59 | 11870.42 | ✅ | +| c128_i1024_o256 | 128 | 1024 | 256 | 161.16 | 2560 | 0 | 15.89 | 16266.52 | 3250.72 | 19517.24 | 259.69 | 356.25 | 2333.30 | 36.68 | 40.81 | 51.99 | 7727.17 | 10133.95 | 11225.21 | ✅ | +| c128_i1024_o4096 | 128 | 1024 | 4096 | 163.16 | 2560 | 0 | 15.69 | 16066.27 | 3281.03 | 19347.31 | 257.28 | 422.58 | 2303.92 | 36.09 | 39.21 | 40.35 | 7768.07 | 10730.88 | 12234.15 | ✅ | +| c128_i1024_o512 | 128 | 1024 | 512 | 161.49 | 2560 | 0 | 15.85 | 16232.48 | 3275.92 | 19508.40 | 255.16 | 334.67 | 2297.21 | 36.27 | 39.39 | 40.57 | 7716.22 | 10581.99 | 11832.60 | ✅ | +| c128_i16384_o1024 | 128 | 16384 | 1024 | 919.25 | 2560 | 0 | 2.78 | 45627.36 | 697.33 | 46324.69 | 2625.93 | 3630.11 | 33634.45 | 171.30 | 196.69 | 207.71 | 45001.23 | 90438.26 | 123707.69 | ❌ | +| c128_i16384_o128 | 128 | 16384 | 128 | 859.37 | 2560 | 0 | 2.98 | 48806.72 | 380.95 | 49187.68 | 2833.19 | 3131.25 | 33643.84 | 311.88 | 328.04 | 328.10 | 42403.99 | 44239.06 | 73426.74 | ❌ | +| c128_i16384_o2048 | 128 | 16384 | 2048 | 918.61 | 2560 | 0 | 2.79 | 45659.28 | 701.99 | 46361.27 | 2601.36 | 3607.43 | 33624.66 | 170.03 | 195.00 | 208.79 | 45030.71 | 92401.43 | 124284.89 | ❌ | +| c128_i16384_o256 | 128 | 16384 | 256 | 897.62 | 2560 | 0 | 2.85 | 46726.87 | 602.42 | 47329.29 | 2804.71 | 4596.59 | 33617.37 | 197.06 | 224.66 | 236.73 | 44219.84 | 57755.16 | 76587.71 | ❌ | +| c128_i16384_o512 | 128 | 16384 | 512 | 915.13 | 2560 | 0 | 2.80 | 45833.00 | 684.56 | 46517.56 | 2646.03 | 3737.57 | 33718.32 | 174.28 | 203.88 | 224.11 | 44914.54 | 88267.88 | 100313.27 | ❌ | +| c128_i2048_o1024 | 128 | 2048 | 1024 | 204.30 | 2560 | 0 | 12.53 | 25663.20 | 2671.39 | 28334.59 | 372.84 | 493.00 | 4208.03 | 44.24 | 48.11 | 50.53 | 9762.62 | 15935.84 | 21541.33 | ✅ | +| c128_i2048_o128 | 128 | 2048 | 128 | 153.29 | 2560 | 0 | 16.70 | 34202.45 | 2125.54 | 36327.99 | 791.19 | 1283.06 | 4199.42 | 52.51 | 58.15 | 59.97 | 7426.54 | 8226.85 | 10576.91 | ⚠️ | +| c128_i2048_o2048 | 128 | 2048 | 2048 | 208.80 | 2560 | 0 | 12.26 | 25109.31 | 2631.24 | 27740.55 | 365.94 | 530.02 | 4211.56 | 44.40 | 48.85 | 54.00 | 9842.38 | 16285.06 | 20408.42 | ✅ | +| c128_i2048_o256 | 128 | 2048 | 256 | 196.63 | 2560 | 0 | 13.02 | 26663.63 | 2605.54 | 29269.17 | 379.32 | 522.39 | 4205.72 | 45.73 | 50.01 | 51.66 | 9476.10 | 12538.01 | 13720.71 | ✅ | +| c128_i2048_o4096 | 128 | 2048 | 4096 | 204.98 | 2560 | 0 | 12.49 | 25578.03 | 2696.32 | 28274.35 | 369.65 | 497.47 | 4203.66 | 44.06 | 47.90 | 49.49 | 9834.14 | 15705.61 | 22444.28 | ✅ | +| c128_i2048_o512 | 128 | 2048 | 512 | 205.58 | 2560 | 0 | 12.45 | 25502.28 | 2682.24 | 28184.52 | 364.96 | 479.81 | 4198.31 | 44.32 | 48.38 | 49.46 | 9849.41 | 16273.27 | 21537.21 | ✅ | +| c128_i4096_o1024 | 128 | 4096 | 1024 | 299.56 | 2560 | 0 | 8.55 | 35004.04 | 1823.49 | 36827.53 | 670.94 | 898.57 | 8472.05 | 64.95 | 71.72 | 78.31 | 14410.80 | 23823.73 | 34763.30 | ⚠️ | +| c128_i4096_o128 | 128 | 4096 | 128 | 252.68 | 2560 | 0 | 10.13 | 41497.56 | 1283.45 | 42781.02 | 1110.40 | 1837.39 | 8464.33 | 89.25 | 99.28 | 101.99 | 12341.74 | 13909.51 | 19709.73 | ⚠️ | +| c128_i4096_o2048 | 128 | 4096 | 2048 | 299.51 | 2560 | 0 | 8.55 | 35009.54 | 1841.89 | 36851.42 | 673.64 | 903.51 | 8456.42 | 64.31 | 71.20 | 77.98 | 14467.32 | 24917.88 | 37076.16 | ⚠️ | +| c128_i4096_o256 | 128 | 4096 | 256 | 288.98 | 2560 | 0 | 8.86 | 36284.85 | 1743.96 | 38028.81 | 688.49 | 1029.32 | 8474.33 | 68.27 | 74.67 | 80.07 | 14052.64 | 18727.14 | 21844.01 | ⚠️ | +| c128_i4096_o4096 | 128 | 4096 | 4096 | 297.69 | 2560 | 0 | 8.60 | 35223.60 | 1820.84 | 37044.43 | 679.88 | 979.68 | 8471.74 | 65.18 | 72.12 | 79.04 | 14368.53 | 24055.68 | 33864.98 | ⚠️ | +| c128_i4096_o512 | 128 | 4096 | 512 | 296.90 | 2560 | 0 | 8.62 | 35317.89 | 1815.79 | 37133.68 | 666.19 | 888.77 | 8469.32 | 65.45 | 72.23 | 78.81 | 14377.25 | 24981.06 | 33674.28 | ⚠️ | +| c128_i512_o256 | 128 | 512 | 256 | 155.29 | 2560 | 0 | 16.49 | 8440.72 | 3401.04 | 11841.77 | 216.12 | 439.65 | 1351.98 | 35.17 | 38.71 | 39.83 | 7433.74 | 9800.36 | 10039.98 | ✅ | +| c16_i512_o256 | 16 | 512 | 256 | 60.82 | 320 | 0 | 5.26 | 2693.64 | 1089.67 | 3783.32 | 145.56 | 311.20 | 388.94 | 12.62 | 17.00 | 17.83 | 2748.60 | 4105.76 | 4640.23 | ✅ | +| c1_i1024_o1024 | 1 | 1024 | 1024 | 52.34 | 20 | 0 | 0.38 | 391.29 | 85.14 | 476.42 | 115.81 | 118.26 | 140.50 | 6.99 | 7.00 | 7.00 | 1666.63 | 2301.13 | 2358.69 | ✅ | +| c1_i1024_o128 | 1 | 1024 | 128 | 32.71 | 20 | 0 | 0.61 | 626.06 | 78.07 | 704.13 | 115.30 | 122.94 | 128.47 | 6.96 | 6.97 | 7.01 | 997.08 | 1004.84 | 1011.97 | ✅ | +| c1_i1024_o2048 | 1 | 1024 | 2048 | 48.50 | 20 | 0 | 0.41 | 422.30 | 83.02 | 505.31 | 115.62 | 119.50 | 126.29 | 6.99 | 7.00 | 7.01 | 1516.37 | 1700.77 | 1778.35 | ✅ | +| c1_i1024_o256 | 1 | 1024 | 256 | 48.48 | 20 | 0 | 0.41 | 422.40 | 81.26 | 503.66 | 114.81 | 120.58 | 125.23 | 6.99 | 7.01 | 7.01 | 1485.39 | 1837.55 | 1880.36 | ✅ | +| c1_i1024_o4096 | 1 | 1024 | 4096 | 51.98 | 20 | 0 | 0.38 | 393.97 | 82.72 | 476.69 | 116.51 | 126.81 | 137.83 | 6.99 | 7.00 | 7.01 | 1612.52 | 2093.77 | 2163.97 | ✅ | +| c1_i1024_o512 | 1 | 1024 | 512 | 49.68 | 20 | 0 | 0.40 | 412.23 | 78.90 | 491.13 | 132.68 | 174.48 | 175.39 | 7.00 | 7.02 | 7.02 | 1497.09 | 2052.55 | 2150.57 | ✅ | +| c1_i131072_o128 | 1 | 131072 | 128 | 257.45 | 20 | 0 | 0.08 | 10182.44 | 8.90 | 10191.34 | 6471.48 | 6574.88 | 6590.68 | 7.21 | 7.25 | 7.32 | 7290.28 | 7408.33 | 7425.83 | ⚠️ | +| c1_i131072_o256 | 1 | 131072 | 256 | 259.43 | 20 | 0 | 0.08 | 10104.51 | 13.11 | 10117.61 | 6523.80 | 6605.21 | 6683.36 | 7.25 | 7.28 | 7.30 | 7749.51 | 8208.24 | 8219.55 | ⚠️ | +| c1_i131072_o512 | 1 | 131072 | 512 | 281.03 | 20 | 0 | 0.07 | 9328.01 | 15.40 | 9343.41 | 6520.17 | 6595.29 | 6639.39 | 7.26 | 7.27 | 7.29 | 8083.13 | 9930.97 | 10147.71 | ⚠️ | +| c1_i16384_o1024 | 1 | 16384 | 1024 | 85.55 | 20 | 0 | 0.23 | 3830.19 | 66.12 | 3896.31 | 761.59 | 767.17 | 767.57 | 7.09 | 7.11 | 7.11 | 2761.83 | 4275.72 | 5529.53 | ✅ | +| c1_i16384_o128 | 1 | 16384 | 128 | 54.06 | 20 | 0 | 0.37 | 6061.09 | 46.98 | 6108.07 | 758.98 | 762.04 | 764.60 | 7.03 | 7.04 | 7.04 | 1645.31 | 1649.67 | 1651.54 | ✅ | +| c1_i16384_o2048 | 1 | 16384 | 2048 | 90.20 | 20 | 0 | 0.22 | 3632.96 | 62.71 | 3695.67 | 762.25 | 769.13 | 769.49 | 7.10 | 7.11 | 7.11 | 2762.54 | 3858.26 | 4063.08 | ✅ | +| c1_i16384_o256 | 1 | 16384 | 256 | 73.13 | 20 | 0 | 0.27 | 4480.86 | 56.19 | 4537.04 | 757.24 | 769.18 | 772.94 | 7.09 | 7.11 | 7.11 | 2205.64 | 2558.68 | 2568.89 | ✅ | +| c1_i16384_o512 | 1 | 16384 | 512 | 75.43 | 20 | 0 | 0.27 | 4344.23 | 53.96 | 4398.19 | 761.98 | 768.30 | 778.23 | 7.10 | 7.11 | 7.11 | 2198.96 | 2832.27 | 2889.46 | ✅ | +| c1_i2048_o1024 | 1 | 2048 | 1024 | 52.15 | 20 | 0 | 0.38 | 785.38 | 81.93 | 867.31 | 135.15 | 140.23 | 150.82 | 7.07 | 7.09 | 7.10 | 1639.00 | 2263.67 | 3129.84 | ✅ | +| c1_i2048_o128 | 1 | 2048 | 128 | 33.24 | 20 | 0 | 0.60 | 1232.22 | 74.97 | 1307.19 | 136.94 | 159.17 | 181.96 | 7.04 | 7.10 | 7.10 | 1006.62 | 1053.40 | 1075.84 | ✅ | +| c1_i2048_o2048 | 1 | 2048 | 2048 | 51.48 | 20 | 0 | 0.39 | 795.71 | 84.60 | 880.31 | 134.55 | 137.97 | 148.69 | 7.07 | 7.09 | 7.10 | 1667.40 | 3179.96 | 3366.92 | ✅ | +| c1_i2048_o256 | 1 | 2048 | 256 | 48.45 | 20 | 0 | 0.41 | 845.46 | 79.01 | 924.47 | 133.36 | 134.84 | 138.71 | 7.07 | 7.08 | 7.09 | 1479.65 | 1930.94 | 1931.75 | ✅ | +| c1_i2048_o4096 | 1 | 2048 | 4096 | 52.65 | 20 | 0 | 0.38 | 777.92 | 77.54 | 855.46 | 134.26 | 137.78 | 149.15 | 7.08 | 7.09 | 7.10 | 1571.48 | 2033.68 | 2095.42 | ✅ | +| c1_i2048_o512 | 1 | 2048 | 512 | 50.39 | 20 | 0 | 0.40 | 812.88 | 79.10 | 891.98 | 133.86 | 135.40 | 147.37 | 7.08 | 7.10 | 7.10 | 1537.04 | 2769.40 | 2894.64 | ✅ | +| c1_i262144_o256 | 1 | 262144 | 256 | 556.90 | 20 | 0 | 0.04 | 9414.44 | 6.27 | 9420.71 | 15234.61 | 15316.53 | 15320.27 | 7.47 | 7.55 | 7.57 | 16530.72 | 17025.05 | 17032.91 | ⚠️ | +| c1_i4096_o1024 | 1 | 4096 | 1024 | 50.77 | 20 | 0 | 0.39 | 1613.53 | 78.27 | 1691.81 | 223.98 | 228.84 | 229.98 | 7.08 | 7.09 | 7.09 | 1622.74 | 2090.63 | 2641.63 | ✅ | +| c1_i4096_o128 | 1 | 4096 | 128 | 36.18 | 20 | 0 | 0.55 | 2264.03 | 70.17 | 2334.20 | 223.37 | 225.71 | 227.21 | 7.03 | 7.04 | 7.07 | 1108.62 | 1117.89 | 1119.93 | ✅ | +| c1_i4096_o2048 | 1 | 4096 | 2048 | 54.09 | 20 | 0 | 0.37 | 1514.64 | 78.34 | 1592.98 | 223.30 | 226.85 | 227.97 | 7.08 | 7.09 | 7.09 | 1715.15 | 2736.41 | 2965.24 | ✅ | +| c1_i4096_o256 | 1 | 4096 | 256 | 53.93 | 20 | 0 | 0.37 | 1519.00 | 77.17 | 1596.18 | 224.11 | 227.58 | 229.35 | 7.07 | 7.09 | 7.09 | 1687.85 | 2022.93 | 2023.17 | ✅ | +| c1_i4096_o4096 | 1 | 4096 | 4096 | 50.60 | 20 | 0 | 0.40 | 1618.96 | 72.98 | 1691.94 | 223.43 | 228.43 | 229.73 | 7.08 | 7.09 | 7.09 | 1522.86 | 2146.74 | 2282.49 | ✅ | +| c1_i4096_o512 | 1 | 4096 | 512 | 59.75 | 20 | 0 | 0.33 | 1371.10 | 81.14 | 1452.25 | 224.35 | 229.07 | 230.84 | 7.07 | 7.08 | 7.08 | 1931.53 | 2977.28 | 3648.36 | ✅ | +| c1_i512_o256 | 1 | 512 | 256 | 53.26 | 20 | 0 | 0.38 | 192.26 | 80.81 | 273.07 | 167.28 | 203.13 | 205.33 | 6.99 | 7.02 | 7.03 | 1663.97 | 1951.39 | 1953.26 | ✅ | +| c1_i524288_o128 | 1 | 524288 | 128 | 1513.75 | 20 | 0 | 0.01 | 6927.01 | 1.20 | 6928.22 | 40602.84 | 41121.70 | 41219.49 | 7.78 | 7.92 | 7.93 | 41303.57 | 41857.14 | 41958.45 | ⚠️ | +| c1_i65536_o1024 | 1 | 65536 | 1024 | 152.81 | 20 | 0 | 0.13 | 8577.64 | 26.28 | 8603.92 | 2984.49 | 2994.56 | 3006.30 | 7.18 | 7.18 | 7.19 | 4418.27 | 5282.47 | 5593.15 | ✅ | +| c1_i65536_o128 | 1 | 65536 | 128 | 134.86 | 20 | 0 | 0.15 | 9718.82 | 17.89 | 9736.71 | 2996.65 | 3058.30 | 3078.26 | 7.11 | 7.16 | 7.16 | 3847.83 | 3911.31 | 3932.26 | ⚠️ | +| c1_i65536_o256 | 1 | 65536 | 256 | 153.56 | 20 | 0 | 0.13 | 8535.68 | 25.96 | 8561.64 | 2986.14 | 2994.62 | 3008.90 | 7.16 | 7.18 | 7.21 | 4407.24 | 4762.51 | 4778.90 | ✅ | +| c1_i65536_o512 | 1 | 65536 | 512 | 157.16 | 20 | 0 | 0.13 | 8340.12 | 27.70 | 8367.82 | 2986.06 | 2996.34 | 3023.15 | 7.18 | 7.19 | 7.20 | 4541.31 | 6107.66 | 6492.71 | ✅ | +| c2_i512_o256 | 2 | 512 | 256 | 44.94 | 40 | 0 | 0.89 | 455.73 | 188.32 | 644.06 | 123.41 | 177.91 | 187.26 | 7.06 | 7.48 | 7.68 | 1611.58 | 2020.21 | 2050.24 | ✅ | +| c32_i1024_o1024 | 32 | 1024 | 1024 | 78.93 | 640 | 0 | 8.11 | 8303.55 | 1684.20 | 9987.75 | 154.93 | 217.27 | 815.30 | 16.71 | 18.61 | 18.94 | 3611.27 | 4919.69 | 5510.54 | ✅ | +| c32_i1024_o128 | 32 | 1024 | 128 | 46.47 | 640 | 0 | 13.77 | 14103.72 | 1762.90 | 15866.62 | 408.25 | 714.24 | 757.98 | 13.41 | 16.99 | 17.83 | 2110.85 | 2431.17 | 2650.42 | ✅ | +| c32_i1024_o2048 | 32 | 1024 | 2048 | 81.00 | 640 | 0 | 7.90 | 8090.92 | 1622.21 | 9713.13 | 162.95 | 334.37 | 753.35 | 17.45 | 23.66 | 25.29 | 3730.41 | 5542.76 | 6244.95 | ✅ | +| c32_i1024_o256 | 32 | 1024 | 256 | 76.79 | 640 | 0 | 8.33 | 8534.95 | 1672.11 | 10207.06 | 154.72 | 212.02 | 753.62 | 16.97 | 19.11 | 20.00 | 3544.98 | 4746.50 | 4887.24 | ✅ | +| c32_i1024_o4096 | 32 | 1024 | 4096 | 77.85 | 640 | 0 | 8.22 | 8418.72 | 1664.78 | 10083.50 | 154.28 | 214.19 | 756.53 | 16.85 | 18.82 | 19.49 | 3549.62 | 4954.17 | 5479.06 | ✅ | +| c32_i1024_o512 | 32 | 1024 | 512 | 77.73 | 640 | 0 | 8.23 | 8431.17 | 1700.51 | 10131.68 | 155.17 | 216.90 | 751.85 | 16.62 | 18.26 | 18.67 | 3574.12 | 4831.87 | 5478.82 | ✅ | +| c32_i16384_o1024 | 32 | 16384 | 1024 | 277.96 | 640 | 0 | 2.30 | 37723.90 | 563.76 | 38287.65 | 1266.27 | 1984.25 | 8777.02 | 48.75 | 60.64 | 65.85 | 13073.64 | 26643.75 | 35597.74 | ✅ | +| c32_i16384_o128 | 32 | 16384 | 128 | 240.94 | 640 | 0 | 2.66 | 43520.72 | 338.90 | 43859.62 | 3738.21 | 4848.25 | 8794.30 | 61.61 | 81.93 | 82.76 | 11539.01 | 13150.69 | 16925.01 | ❌ | +| c32_i16384_o2048 | 32 | 16384 | 2048 | 277.48 | 640 | 0 | 2.31 | 37789.10 | 576.94 | 38366.04 | 1268.99 | 2250.82 | 8796.37 | 47.57 | 58.28 | 64.83 | 13129.88 | 24681.93 | 38211.25 | ✅ | +| c32_i16384_o256 | 32 | 16384 | 256 | 264.70 | 640 | 0 | 2.42 | 39614.37 | 508.66 | 40123.04 | 1298.03 | 1994.66 | 8761.72 | 54.06 | 64.54 | 70.67 | 12625.00 | 16418.68 | 19908.81 | ⚠️ | +| c32_i16384_o512 | 32 | 16384 | 512 | 275.93 | 640 | 0 | 2.32 | 38001.51 | 561.14 | 38562.65 | 1268.67 | 1990.86 | 8787.03 | 48.89 | 60.15 | 66.01 | 13034.17 | 25370.34 | 27540.55 | ✅ | +| c32_i2048_o1024 | 32 | 2048 | 1024 | 87.95 | 640 | 0 | 7.28 | 14903.68 | 1574.85 | 16478.54 | 200.89 | 270.09 | 1352.10 | 17.77 | 19.91 | 20.51 | 4023.46 | 6506.86 | 8623.71 | ✅ | +| c32_i2048_o128 | 32 | 2048 | 128 | 59.24 | 640 | 0 | 10.80 | 22125.82 | 1375.94 | 23501.76 | 367.53 | 672.56 | 1352.00 | 18.71 | 21.55 | 22.09 | 2732.47 | 3094.92 | 3404.73 | ✅ | +| c32_i2048_o2048 | 32 | 2048 | 2048 | 89.13 | 640 | 0 | 7.18 | 14705.49 | 1556.10 | 16261.59 | 199.91 | 291.10 | 1361.13 | 17.87 | 20.28 | 20.98 | 4056.53 | 6825.74 | 9339.78 | ✅ | +| c32_i2048_o256 | 32 | 2048 | 256 | 83.40 | 640 | 0 | 7.67 | 15715.67 | 1544.12 | 17259.80 | 202.63 | 380.57 | 1355.98 | 18.19 | 20.18 | 20.73 | 3838.39 | 5110.33 | 5339.24 | ✅ | +| c32_i2048_o4096 | 32 | 2048 | 4096 | 88.69 | 640 | 0 | 7.22 | 14778.01 | 1520.45 | 16298.46 | 199.19 | 288.11 | 1360.76 | 18.06 | 20.41 | 20.84 | 3982.98 | 6383.30 | 9405.39 | ✅ | +| c32_i2048_o512 | 32 | 2048 | 512 | 90.84 | 640 | 0 | 7.05 | 14428.41 | 1530.32 | 15958.73 | 197.17 | 253.30 | 1349.46 | 17.88 | 20.25 | 21.02 | 4060.40 | 6700.48 | 9080.73 | ✅ | +| c32_i4096_o1024 | 32 | 4096 | 1024 | 113.81 | 640 | 0 | 5.62 | 23032.89 | 1212.69 | 24245.58 | 339.67 | 542.37 | 2422.62 | 22.72 | 26.10 | 27.21 | 5215.72 | 8904.34 | 13457.62 | ✅ | +| c32_i4096_o128 | 32 | 4096 | 128 | 85.66 | 640 | 0 | 7.47 | 30604.61 | 950.87 | 31555.48 | 556.18 | 1180.23 | 2449.13 | 27.29 | 31.52 | 34.82 | 4002.80 | 4613.66 | 5507.05 | ✅ | +| c32_i4096_o2048 | 32 | 4096 | 2048 | 114.27 | 640 | 0 | 5.60 | 22939.90 | 1189.49 | 24129.39 | 341.11 | 550.60 | 2435.46 | 22.78 | 26.02 | 27.73 | 5165.78 | 9243.12 | 13018.90 | ✅ | +| c32_i4096_o256 | 32 | 4096 | 256 | 106.21 | 640 | 0 | 6.03 | 24681.91 | 1178.61 | 25860.52 | 350.79 | 564.37 | 2447.99 | 23.55 | 26.71 | 28.32 | 4937.54 | 6734.16 | 7405.49 | ✅ | +| c32_i4096_o4096 | 32 | 4096 | 4096 | 114.94 | 640 | 0 | 5.57 | 22806.49 | 1192.22 | 23998.71 | 341.86 | 550.55 | 2448.39 | 22.83 | 26.16 | 27.23 | 5203.63 | 8545.93 | 12625.46 | ✅ | +| c32_i4096_o512 | 32 | 4096 | 512 | 112.20 | 640 | 0 | 5.70 | 23364.93 | 1200.40 | 24565.33 | 340.20 | 560.25 | 2385.06 | 22.99 | 26.29 | 27.73 | 5150.64 | 8529.09 | 12164.94 | ✅ | +| c32_i512_o256 | 32 | 512 | 256 | 78.27 | 640 | 0 | 8.18 | 4186.51 | 1690.88 | 5877.39 | 149.00 | 289.93 | 473.49 | 16.81 | 19.27 | 22.36 | 3610.39 | 4786.53 | 5755.87 | ✅ | +| c32_i65536_o1024 | 32 | 65536 | 1024 | 266.52 | 160 | 0 | 0.60 | 39343.10 | 131.31 | 39474.42 | 8656.40 | 33860.81 | 43057.06 | 171.14 | 244.27 | 281.95 | 46303.99 | 103438.83 | 127750.75 | ❌ | +| c4_i512_o256 | 4 | 512 | 256 | 43.40 | 80 | 0 | 1.84 | 943.85 | 379.18 | 1323.03 | 118.07 | 125.54 | 241.46 | 7.83 | 8.29 | 8.68 | 1720.27 | 2165.16 | 2253.32 | ✅ | +| c64_i512_o256 | 64 | 512 | 256 | 109.33 | 1280 | 0 | 11.71 | 5994.26 | 2402.36 | 8396.62 | 171.45 | 299.06 | 771.51 | 24.39 | 26.72 | 27.58 | 5151.60 | 6720.29 | 6878.42 | ✅ | +| c8_i1024_o1024 | 8 | 1024 | 1024 | 47.80 | 160 | 0 | 3.35 | 3427.67 | 696.10 | 4123.77 | 129.12 | 192.22 | 281.04 | 9.27 | 9.95 | 10.24 | 2052.01 | 2788.77 | 3052.14 | ✅ | +| c8_i1024_o128 | 8 | 1024 | 128 | 31.03 | 160 | 0 | 5.16 | 5279.38 | 659.92 | 5939.30 | 187.55 | 269.21 | 292.69 | 8.88 | 9.90 | 9.97 | 1315.49 | 1387.68 | 1458.94 | ✅ | +| c8_i1024_o2048 | 8 | 1024 | 2048 | 47.63 | 160 | 0 | 3.36 | 3439.84 | 684.76 | 4124.59 | 139.64 | 191.21 | 283.85 | 9.49 | 10.93 | 11.77 | 2064.28 | 2867.60 | 3177.60 | ✅ | +| c8_i1024_o256 | 8 | 1024 | 256 | 47.68 | 160 | 0 | 3.36 | 3436.34 | 684.94 | 4121.28 | 134.22 | 195.34 | 290.76 | 9.44 | 10.83 | 11.66 | 2050.72 | 2669.20 | 2870.35 | ✅ | +| c8_i1024_o4096 | 8 | 1024 | 4096 | 48.02 | 160 | 0 | 3.33 | 3411.81 | 683.42 | 4095.23 | 132.59 | 194.92 | 285.39 | 9.39 | 10.45 | 11.05 | 2050.12 | 2647.51 | 2950.32 | ✅ | +| c8_i1024_o512 | 8 | 1024 | 512 | 48.89 | 160 | 0 | 3.27 | 3351.35 | 699.85 | 4051.20 | 128.43 | 130.82 | 286.17 | 9.25 | 9.96 | 10.23 | 2097.31 | 2846.33 | 3059.68 | ✅ | +| c8_i131072_o512 | 8 | 131072 | 512 | 593.71 | 160 | 0 | 0.27 | 35322.95 | 49.97 | 35372.92 | 8837.26 | 12856.65 | 21274.84 | 95.98 | 174.74 | 250.31 | 26223.81 | 43791.96 | 52427.10 | ❌ | +| c8_i16384_o1024 | 8 | 16384 | 1024 | 117.03 | 160 | 0 | 1.37 | 22399.35 | 381.47 | 22780.82 | 880.55 | 1325.49 | 2519.24 | 15.14 | 18.88 | 22.89 | 5043.96 | 9664.45 | 11942.89 | ✅ | +| c8_i16384_o128 | 8 | 16384 | 128 | 83.84 | 160 | 0 | 1.91 | 31265.56 | 241.22 | 31506.78 | 1766.89 | 2637.47 | 2676.54 | 14.95 | 23.25 | 26.08 | 3640.27 | 3713.62 | 5332.84 | ✅ | +| c8_i16384_o2048 | 8 | 16384 | 2048 | 111.65 | 160 | 0 | 1.43 | 23479.38 | 353.00 | 23832.38 | 858.33 | 1105.07 | 2515.80 | 15.80 | 19.89 | 21.08 | 4778.32 | 8910.12 | 13501.67 | ✅ | +| c8_i16384_o256 | 8 | 16384 | 256 | 101.77 | 160 | 0 | 1.57 | 25759.71 | 329.17 | 26088.88 | 867.77 | 1267.33 | 2511.92 | 17.06 | 21.93 | 23.13 | 4427.86 | 5958.68 | 6667.56 | ✅ | +| c8_i16384_o512 | 8 | 16384 | 512 | 109.64 | 160 | 0 | 1.46 | 23908.84 | 355.69 | 24264.53 | 908.35 | 1361.76 | 2515.33 | 15.82 | 20.35 | 22.15 | 4755.66 | 8684.67 | 10122.99 | ✅ | +| c8_i2048_o1024 | 8 | 2048 | 1024 | 53.11 | 160 | 0 | 3.01 | 6170.36 | 677.84 | 6848.20 | 150.15 | 153.04 | 410.55 | 9.49 | 10.24 | 10.65 | 2286.09 | 3870.04 | 5178.54 | ✅ | +| c8_i2048_o128 | 8 | 2048 | 128 | 33.35 | 160 | 0 | 4.80 | 9826.22 | 612.40 | 10438.62 | 228.93 | 388.77 | 404.46 | 9.33 | 10.33 | 11.02 | 1411.23 | 1530.95 | 1595.88 | ✅ | +| c8_i2048_o2048 | 8 | 2048 | 2048 | 51.12 | 160 | 0 | 3.13 | 6409.90 | 672.70 | 7082.60 | 150.70 | 165.44 | 418.00 | 9.56 | 10.25 | 10.68 | 2202.49 | 3406.00 | 4277.62 | ✅ | +| c8_i2048_o256 | 8 | 2048 | 256 | 48.03 | 160 | 0 | 3.33 | 6822.11 | 658.85 | 7480.96 | 148.57 | 147.18 | 403.31 | 9.66 | 10.48 | 10.98 | 2051.77 | 2715.99 | 2745.12 | ✅ | +| c8_i2048_o4096 | 8 | 2048 | 4096 | 53.12 | 160 | 0 | 3.01 | 6168.50 | 661.05 | 6829.55 | 150.99 | 219.64 | 421.18 | 9.58 | 10.44 | 10.58 | 2250.24 | 3788.73 | 4649.51 | ✅ | +| c8_i2048_o512 | 8 | 2048 | 512 | 50.97 | 160 | 0 | 3.14 | 6428.69 | 669.04 | 7097.73 | 148.41 | 147.71 | 403.41 | 9.60 | 10.48 | 11.08 | 2190.12 | 3392.37 | 4269.13 | ✅ | +| c8_i262144_o128 | 8 | 262144 | 128 | 1333.56 | 160 | 0 | 0.12 | 31451.94 | 12.61 | 31464.55 | 36128.45 | 57997.25 | 58592.27 | 215.95 | 392.14 | 446.94 | 59143.65 | 61777.27 | 74358.75 | ❌ | +| c8_i4096_o1024 | 8 | 4096 | 1024 | 60.18 | 160 | 0 | 2.66 | 10890.64 | 578.38 | 11469.02 | 253.10 | 364.29 | 731.93 | 10.57 | 11.89 | 12.17 | 2550.26 | 4642.49 | 7535.89 | ✅ | +| c8_i4096_o128 | 8 | 4096 | 128 | 40.35 | 160 | 0 | 3.96 | 16240.24 | 506.02 | 16746.26 | 376.51 | 695.40 | 727.26 | 10.62 | 11.99 | 13.73 | 1721.35 | 1910.10 | 2181.08 | ✅ | +| c8_i4096_o2048 | 8 | 4096 | 2048 | 57.14 | 160 | 0 | 2.80 | 11470.28 | 593.22 | 12063.50 | 254.35 | 361.27 | 732.14 | 10.52 | 11.68 | 12.36 | 2478.80 | 3843.55 | 5407.45 | ✅ | +| c8_i4096_o256 | 8 | 4096 | 256 | 54.73 | 160 | 0 | 2.92 | 11973.53 | 575.24 | 12548.76 | 252.76 | 366.29 | 732.21 | 10.63 | 12.02 | 12.59 | 2342.56 | 3122.99 | 3247.98 | ✅ | +| c8_i4096_o4096 | 8 | 4096 | 4096 | 61.97 | 160 | 0 | 2.58 | 10574.93 | 602.60 | 11177.53 | 251.62 | 276.31 | 733.83 | 10.48 | 11.81 | 12.23 | 2715.50 | 3797.26 | 5487.12 | ✅ | +| c8_i4096_o512 | 8 | 4096 | 512 | 57.59 | 160 | 0 | 2.78 | 11379.47 | 586.75 | 11966.22 | 255.02 | 376.23 | 733.95 | 10.54 | 11.70 | 12.17 | 2482.90 | 4048.32 | 5636.75 | ✅ | +| c8_i512_o256 | 8 | 512 | 256 | 48.34 | 160 | 0 | 3.31 | 1694.53 | 687.00 | 2381.52 | 125.68 | 194.69 | 258.22 | 9.43 | 10.43 | 10.95 | 2075.15 | 2669.68 | 2871.40 | ✅ | +| c8_i65536_o1024 | 8 | 65536 | 1024 | 298.66 | 160 | 0 | 0.54 | 35109.32 | 107.56 | 35216.88 | 3847.52 | 5884.53 | 9844.52 | 46.16 | 82.22 | 101.31 | 13001.39 | 21336.69 | 30533.51 | ⚠️ | +| c8_i65536_o128 | 8 | 65536 | 128 | 274.11 | 160 | 0 | 0.58 | 38254.29 | 70.30 | 38324.59 | 7223.66 | 11101.75 | 11262.28 | 40.50 | 72.54 | 72.63 | 12089.84 | 12347.21 | 12881.32 | ⚠️ | +| c8_i65536_o256 | 8 | 65536 | 256 | 291.01 | 160 | 0 | 0.55 | 36032.88 | 98.87 | 36131.75 | 3804.89 | 5718.15 | 9852.36 | 50.92 | 78.36 | 96.49 | 12843.24 | 18584.33 | 21293.29 | ❌ | +| c8_i65536_o512 | 8 | 65536 | 512 | 297.04 | 160 | 0 | 0.54 | 35301.20 | 111.90 | 35413.10 | 3902.29 | 6055.87 | 9835.60 | 44.23 | 73.93 | 83.82 | 13139.03 | 22983.14 | 31590.46 | ⚠️ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json new file mode 100644 index 0000000..4f3bbb4 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json @@ -0,0 +1,6429 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp4_custom_bench", + "run_id": "20260711-133908", + "timestamp": "2026-07-11T13:39:08+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "def1355", + "git_dirty": "dirty", + "description": "H200 2x vLLM TP=4 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 4, + "num_services": 2, + "ports": "30005,30006", + "gpu_groups": "0,1,2,3|4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "2 512 256 40", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [ + { + "name": "c128_i1024_o1024", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 172.39323008002248, + "request_throughput": 14.849771065903717, + "input_token_throughput": 15206.165571485406, + "output_token_throughput": 3100.5973944097605, + "total_token_throughput": 18306.762965895166, + "total_input_tokens": 2621440, + "total_output_tokens": 534522, + "e2e_ms": { + "mean": 8264.996455389724, + "p50": 7789.678987493971, + "p90": 11756.317305739502, + "p95": 13967.23252292723, + "p99": 17536.458535679838 + }, + "ttft_ms": { + "mean": 267.1602980701664, + "p50": 194.55134301097132, + "p90": 328.0344645609148, + "p95": 478.8006908784156, + "p99": 2297.698974920786 + }, + "tpot_ms": { + "mean": 38.53003130054492, + "p50": 36.742621067679, + "p90": 59.41429369279392, + "p95": 66.1565177263384, + "p99": 70.07324518675013 + }, + "itl_ms": { + "mean": 38.33531172431711, + "p50": 36.54410010322313, + "p90": 59.16513506822693, + "p95": 65.93954884337958, + "p99": 69.76542250785654 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 98.83996513002785, + "request_throughput": 25.900454301377177, + "input_token_throughput": 26522.06520461023, + "output_token_throughput": 3314.9141601675883, + "total_token_throughput": 29836.979364777817, + "total_input_tokens": 2621440, + "total_output_tokens": 327646, + "e2e_ms": { + "mean": 4730.889936950962, + "p50": 4738.875214505242, + "p90": 5273.3935622207355, + "p95": 5530.639346732642, + "p99": 6122.148045647769 + }, + "ttft_ms": { + "mean": 900.3881096975647, + "p50": 878.4408235223964, + "p90": 1408.8249116961379, + "p95": 1590.8718543098075, + "p99": 2307.9329636477605 + }, + "tpot_ms": { + "mean": 30.164361933300313, + "p50": 30.757310448716737, + "p90": 35.303492756518274, + "p95": 35.98104450527641, + "p99": 37.085646242618296 + }, + "itl_ms": { + "mean": 30.16279034305132, + "p50": 30.757079614091694, + "p90": 35.303211258999326, + "p95": 35.9806825589264, + "p99": 37.08532446034263 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_128.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 161.76455012900988, + "request_throughput": 15.825469782831641, + "input_token_throughput": 16205.2810576196, + "output_token_throughput": 3297.7435388319536, + "total_token_throughput": 19503.024596451552, + "total_input_tokens": 2621440, + "total_output_tokens": 533458, + "e2e_ms": { + "mean": 7732.00934715162, + "p50": 7706.126845994731, + "p90": 9936.003248777706, + "p95": 10616.589111639772, + "p99": 11870.42382700427 + }, + "ttft_ms": { + "mean": 256.6061660457535, + "p50": 183.6360429879278, + "p90": 278.1436800782103, + "p95": 340.86536000250027, + "p99": 2315.9884250303726 + }, + "tpot_ms": { + "mean": 36.0673758854372, + "p50": 36.71776657082684, + "p90": 38.754329601805686, + "p95": 39.269018607056736, + "p99": 40.296127494022976 + }, + "itl_ms": { + "mean": 35.875113844333285, + "p50": 36.535086806523225, + "p90": 38.54275344150884, + "p95": 39.06941631082252, + "p99": 40.037952805839154 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 161.15555513004074, + "request_throughput": 15.885273070073616, + "input_token_throughput": 16266.519623755383, + "output_token_throughput": 3250.7163626923966, + "total_token_throughput": 19517.235986447777, + "total_input_tokens": 2621440, + "total_output_tokens": 523871, + "e2e_ms": { + "mean": 7727.166688055763, + "p50": 7802.593892032746, + "p90": 9742.186820518691, + "p95": 10133.950301926234, + "p99": 11225.207591401988 + }, + "ttft_ms": { + "mean": 259.6949904462008, + "p50": 185.23747954168357, + "p90": 282.4472953216173, + "p95": 356.24627698853345, + "p99": 2333.3000153169146 + }, + "tpot_ms": { + "mean": 36.67608677964262, + "p50": 37.08397564475155, + "p90": 39.790116985757045, + "p95": 40.80520354344157, + "p99": 51.98653523430576 + }, + "itl_ms": { + "mean": 36.49728808087508, + "p50": 36.881504635488966, + "p90": 39.64186447351697, + "p95": 40.63193390640941, + "p99": 51.59359547846483 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_256.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 163.1641573330271, + "request_throughput": 15.689720351847237, + "input_token_throughput": 16066.27364029157, + "output_token_throughput": 3281.0330942188916, + "total_token_throughput": 19347.30673451046, + "total_input_tokens": 2621440, + "total_output_tokens": 535347, + "e2e_ms": { + "mean": 7768.0705221036305, + "p50": 7779.976884485222, + "p90": 9995.535276620649, + "p95": 10730.881073744968, + "p99": 12234.15256866486 + }, + "ttft_ms": { + "mean": 257.28251816292413, + "p50": 183.7318860052619, + "p90": 284.0460309351329, + "p95": 422.5844055210541, + "p99": 2303.9169573405525 + }, + "tpot_ms": { + "mean": 36.08731376501835, + "p50": 36.74589582311257, + "p90": 38.635312321757525, + "p95": 39.20972573694193, + "p99": 40.34653792474833 + }, + "itl_ms": { + "mean": 35.90677394448389, + "p50": 36.55651061150313, + "p90": 38.41832325335143, + "p95": 39.02278168168324, + "p99": 40.18393781864987 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 161.4934899629443, + "request_throughput": 15.852032181528854, + "input_token_throughput": 16232.480953885546, + "output_token_throughput": 3275.9153333140016, + "total_token_throughput": 19508.39628719955, + "total_input_tokens": 2621440, + "total_output_tokens": 529039, + "e2e_ms": { + "mean": 7716.2186734600255, + "p50": 7756.274058541749, + "p90": 9902.339230419602, + "p95": 10581.986936813337, + "p99": 11832.60301870119 + }, + "ttft_ms": { + "mean": 255.16095905081784, + "p50": 183.0740385048557, + "p90": 275.3751680254936, + "p95": 334.6741058048795, + "p99": 2297.212610503774 + }, + "tpot_ms": { + "mean": 36.27411559738953, + "p50": 36.90445407913109, + "p90": 38.89796617059789, + "p95": 39.38925744135109, + "p99": 40.574045828890846 + }, + "itl_ms": { + "mean": 36.088637286494745, + "p50": 36.715503331749815, + "p90": 38.668252778799605, + "p95": 39.183733808278504, + "p99": 40.430493946190595 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_512.jsonl" + }, + { + "name": "c128_i16384_o1024", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 919.2520269450033, + "request_throughput": 2.7848728367864224, + "input_token_throughput": 45627.356557908744, + "output_token_throughput": 697.3343340132238, + "total_token_throughput": 46324.690891921964, + "total_input_tokens": 41943040, + "total_output_tokens": 641026, + "e2e_ms": { + "mean": 45001.22985192386, + "p50": 38537.9672049894, + "p90": 73411.154619226, + "p95": 90438.25863343383, + "p99": 123707.69347336027 + }, + "ttft_ms": { + "mean": 2625.9279194736337, + "p50": 1610.5919524852652, + "p90": 2643.259297683834, + "p95": 3630.1065946550807, + "p99": 33634.44761375713 + }, + "tpot_ms": { + "mean": 171.29838262030282, + "p50": 174.10042585931788, + "p90": 191.857317470299, + "p95": 196.6860248351331, + "p99": 207.7062569780722 + }, + "itl_ms": { + "mean": 170.48918588362815, + "p50": 173.18544062971623, + "p90": 191.07016889404835, + "p95": 195.60057769858426, + "p99": 207.12842061635322 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_1024.jsonl" + }, + { + "name": "c128_i16384_o128", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 859.3701289989986, + "request_throughput": 2.9789259756816415, + "input_token_throughput": 48806.723185568015, + "output_token_throughput": 380.9545956424342, + "total_token_throughput": 49187.67778121045, + "total_input_tokens": 41943040, + "total_output_tokens": 327381, + "e2e_ms": { + "mean": 42403.99054401498, + "p50": 42587.17669401085, + "p90": 43794.49558101478, + "p95": 44239.059685051325, + "p99": 73426.73798753004 + }, + "ttft_ms": { + "mean": 2833.193887487778, + "p50": 1653.9359024900477, + "p90": 2892.610295308985, + "p95": 3131.2537917227023, + "p99": 33643.84073065702 + }, + "tpot_ms": { + "mean": 311.87951799655093, + "p50": 319.0447975710441, + "p90": 327.9145556648709, + "p95": 328.0359147038855, + "p99": 328.1002778994097 + }, + "itl_ms": { + "mean": 311.8659649233483, + "p50": 319.0443645553172, + "p90": 327.91121197616, + "p95": 328.03315464503453, + "p99": 328.09282922379373 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_128.jsonl" + }, + { + "name": "c128_i16384_o2048", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 918.609304343001, + "request_throughput": 2.7868213264298896, + "input_token_throughput": 45659.28061222731, + "output_token_throughput": 701.9861403006408, + "total_token_throughput": 46361.26675252795, + "total_input_tokens": 41943040, + "total_output_tokens": 644851, + "e2e_ms": { + "mean": 45030.70892128417, + "p50": 38617.496168997604, + "p90": 73778.13227887964, + "p95": 92401.43465625009, + "p99": 124284.89116791805 + }, + "ttft_ms": { + "mean": 2601.3592253009847, + "p50": 1578.1983239867259, + "p90": 2635.9606526442803, + "p95": 3607.430891436526, + "p99": 33624.65822858319 + }, + "tpot_ms": { + "mean": 170.03327446799767, + "p50": 173.23042250405052, + "p90": 190.40878788659077, + "p95": 194.99764928126098, + "p99": 208.7899013259607 + }, + "itl_ms": { + "mean": 169.23891149830678, + "p50": 172.44599117557388, + "p90": 189.46610969006963, + "p95": 194.15233367186605, + "p99": 206.90664604638008 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_2048.jsonl" + }, + { + "name": "c128_i16384_o256", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 897.621502821974, + "request_throughput": 2.8519815890682008, + "input_token_throughput": 46726.8663552934, + "output_token_throughput": 602.4231798146295, + "total_token_throughput": 47329.28953510804, + "total_input_tokens": 41943040, + "total_output_tokens": 540748, + "e2e_ms": { + "mean": 44219.84462311193, + "p50": 44721.17852998781, + "p90": 55222.73939058068, + "p95": 57755.15577839106, + "p99": 76587.71378947243 + }, + "ttft_ms": { + "mean": 2804.707829081053, + "p50": 1716.4993534679525, + "p90": 3034.1674943047105, + "p95": 4596.590572808054, + "p99": 33617.3651075537 + }, + "tpot_ms": { + "mean": 197.05998115773372, + "p50": 201.09523076543266, + "p90": 218.96908414091612, + "p95": 224.66402557030736, + "p99": 236.73202214477985 + }, + "itl_ms": { + "mean": 196.31561704005404, + "p50": 200.32954269678584, + "p90": 217.82939194011792, + "p95": 223.82267243435942, + "p99": 236.3526946892711 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_256.jsonl" + }, + { + "name": "c128_i16384_o512", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 915.1275745470193, + "request_throughput": 2.7974241747301507, + "input_token_throughput": 45832.99767877879, + "output_token_throughput": 684.5581068957425, + "total_token_throughput": 46517.55578567453, + "total_input_tokens": 41943040, + "total_output_tokens": 626458, + "e2e_ms": { + "mean": 44914.537590626765, + "p50": 39495.58882598649, + "p90": 74920.82639939617, + "p95": 88267.88074479846, + "p99": 100313.26691627383 + }, + "ttft_ms": { + "mean": 2646.0264690055737, + "p50": 1644.704559526872, + "p90": 2713.0657975503705, + "p95": 3737.5678659940886, + "p99": 33718.316934909584 + }, + "tpot_ms": { + "mean": 174.27917813481235, + "p50": 176.57178472857964, + "p90": 196.3337238559724, + "p95": 203.88145810190872, + "p99": 224.1069659303989 + }, + "itl_ms": { + "mean": 173.4964124988034, + "p50": 175.92428582230116, + "p90": 195.45350120960254, + "p95": 203.05172497723345, + "p99": 222.28400312181145 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_512.jsonl" + }, + { + "name": "c128_i2048_o1024", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 204.29565773101058, + "request_throughput": 12.530858601853733, + "input_token_throughput": 25663.198416596446, + "output_token_throughput": 2671.3930489828444, + "total_token_throughput": 28334.59146557929, + "total_input_tokens": 5242880, + "total_output_tokens": 545754, + "e2e_ms": { + "mean": 9762.615562429983, + "p50": 9157.230406504823, + "p90": 13910.17175068846, + "p95": 15935.838111440538, + "p99": 21541.334415340087 + }, + "ttft_ms": { + "mean": 372.843854514349, + "p50": 242.41412596893497, + "p90": 392.64738988131285, + "p95": 493.00015218323176, + "p99": 4208.031118083745 + }, + "tpot_ms": { + "mean": 44.23840402224793, + "p50": 44.99556358921123, + "p90": 47.38613093340097, + "p95": 48.11152710137389, + "p99": 50.5311058701371 + }, + "itl_ms": { + "mean": 44.00457448991474, + "p50": 44.76411662840158, + "p90": 47.14332725574256, + "p95": 47.914761956692956, + "p99": 50.14406648849003 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_1024.jsonl" + }, + { + "name": "c128_i2048_o128", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 153.28961618000176, + "request_throughput": 16.700413660073988, + "input_token_throughput": 34202.44717583153, + "output_token_throughput": 2125.5386249868307, + "total_token_throughput": 36327.98580081836, + "total_input_tokens": 5242880, + "total_output_tokens": 325823, + "e2e_ms": { + "mean": 7426.542717551774, + "p50": 7519.444196979748, + "p90": 7931.7692122771405, + "p95": 8226.8541852216, + "p99": 10576.912148120464 + }, + "ttft_ms": { + "mean": 791.1919670015322, + "p50": 736.483736516675, + "p90": 1221.6598160099238, + "p95": 1283.0562591931086, + "p99": 4199.415099230828 + }, + "tpot_ms": { + "mean": 52.51428230122135, + "p50": 53.82101512220725, + "p90": 57.72905067094294, + "p95": 58.15436926695123, + "p99": 59.97293118720664 + }, + "itl_ms": { + "mean": 52.48190387140439, + "p50": 53.81724940943825, + "p90": 57.726806252012324, + "p95": 58.15410469809921, + "p99": 59.97246126387649 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_128.jsonl" + }, + { + "name": "c128_i2048_o2048", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 208.80225611501373, + "request_throughput": 12.260403922981968, + "input_token_throughput": 25109.30723426707, + "output_token_throughput": 2631.24072614125, + "total_token_throughput": 27740.54796040832, + "total_input_tokens": 5242880, + "total_output_tokens": 549409, + "e2e_ms": { + "mean": 9842.37848147145, + "p50": 9232.52340304316, + "p90": 13945.226166764041, + "p95": 16285.06117714278, + "p99": 20408.415713311373 + }, + "ttft_ms": { + "mean": 365.94149188463234, + "p50": 239.27259998163208, + "p90": 353.6969173233956, + "p95": 530.0150224182293, + "p99": 4211.561568837496 + }, + "tpot_ms": { + "mean": 44.40326790287457, + "p50": 44.98828430131604, + "p90": 47.74656392552758, + "p95": 48.85120712544617, + "p99": 53.997538806530045 + }, + "itl_ms": { + "mean": 44.172192415332475, + "p50": 44.73914634153712, + "p90": 47.505158283643, + "p95": 48.59071167813023, + "p99": 53.70061425829482 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_2048.jsonl" + }, + { + "name": "c128_i2048_o256", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 196.63041066098958, + "request_throughput": 13.019349303062256, + "input_token_throughput": 26663.6273726715, + "output_token_throughput": 2605.5379647419063, + "total_token_throughput": 29269.165337413408, + "total_input_tokens": 5242880, + "total_output_tokens": 512328, + "e2e_ms": { + "mean": 9476.103467166013, + "p50": 9467.947298486251, + "p90": 12215.088280598866, + "p95": 12538.011490146164, + "p99": 13720.708791241632 + }, + "ttft_ms": { + "mean": 379.32366461509446, + "p50": 246.58072050078772, + "p90": 404.3491132790223, + "p95": 522.392269448028, + "p99": 4205.7175672921585 + }, + "tpot_ms": { + "mean": 45.73275768638084, + "p50": 46.6100753353155, + "p90": 49.16386275490361, + "p95": 50.008765212349545, + "p99": 51.65800156214224 + }, + "itl_ms": { + "mean": 45.52147672991281, + "p50": 46.38860522058796, + "p90": 48.98359987373919, + "p95": 49.841784776414414, + "p99": 51.40792481203061 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_256.jsonl" + }, + { + "name": "c128_i2048_o4096", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 204.97591403801925, + "request_throughput": 12.48927227383978, + "input_token_throughput": 25578.02961682387, + "output_token_throughput": 2696.3216756164234, + "total_token_throughput": 28274.351292440293, + "total_input_tokens": 5242880, + "total_output_tokens": 552681, + "e2e_ms": { + "mean": 9834.139892057077, + "p50": 9220.157204486895, + "p90": 13741.088272898922, + "p95": 15705.61146168039, + "p99": 22444.27695973601 + }, + "ttft_ms": { + "mean": 369.64516925165753, + "p50": 241.21888101217337, + "p90": 378.9612676715477, + "p95": 497.4736295611367, + "p99": 4203.656943541137 + }, + "tpot_ms": { + "mean": 44.063288383150805, + "p50": 44.78743687857023, + "p90": 47.155220997591535, + "p95": 47.89839114115504, + "p99": 49.488713175876434 + }, + "itl_ms": { + "mean": 43.843357811756334, + "p50": 44.5406747903975, + "p90": 46.96878799674696, + "p95": 47.61324272796249, + "p99": 49.25185357024085 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_4096.jsonl" + }, + { + "name": "c128_i2048_o512", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 205.58478230598848, + "request_throughput": 12.45228353618968, + "input_token_throughput": 25502.276682116466, + "output_token_throughput": 2682.2413303882827, + "total_token_throughput": 28184.51801250475, + "total_input_tokens": 5242880, + "total_output_tokens": 551428, + "e2e_ms": { + "mean": 9849.413630793037, + "p50": 9248.099040007219, + "p90": 14181.897066975942, + "p95": 16273.27139499248, + "p99": 21537.212672340098 + }, + "ttft_ms": { + "mean": 364.9559352597862, + "p50": 238.98277248372324, + "p90": 354.852419818053, + "p95": 479.81080181489244, + "p99": 4198.306379870628 + }, + "tpot_ms": { + "mean": 44.31741461426594, + "p50": 45.06919488338426, + "p90": 47.75500610530777, + "p95": 48.38413202043307, + "p99": 49.462320926734186 + }, + "itl_ms": { + "mean": 44.087402562054, + "p50": 44.8524519679176, + "p90": 47.53244151442521, + "p95": 48.13061339723142, + "p99": 49.245863130187566 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_512.jsonl" + }, + { + "name": "c128_i4096_o1024", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 299.5585728090373, + "request_throughput": 8.545907987190038, + "input_token_throughput": 35004.039115530395, + "output_token_throughput": 1823.4897932572892, + "total_token_throughput": 36827.52890878768, + "total_input_tokens": 10485760, + "total_output_tokens": 546242, + "e2e_ms": { + "mean": 14410.804073699523, + "p50": 13498.310438502813, + "p90": 20445.950364420423, + "p95": 23823.7266023556, + "p99": 34763.29548756821 + }, + "ttft_ms": { + "mean": 670.9425271239525, + "p50": 403.0677000118885, + "p90": 679.926768358564, + "p95": 898.5662415187111, + "p99": 8472.05125531822 + }, + "tpot_ms": { + "mean": 64.95321939396936, + "p50": 66.22150749587814, + "p90": 70.0543531767368, + "p95": 71.7235141039819, + "p99": 78.30801132371384 + }, + "itl_ms": { + "mean": 64.61322426057873, + "p50": 65.90216442502557, + "p90": 69.68458649417813, + "p95": 71.33039946453164, + "p99": 77.652479555507 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_1024.jsonl" + }, + { + "name": "c128_i4096_o128", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 252.68375409400323, + "request_throughput": 10.131240962359735, + "input_token_throughput": 41497.562981825475, + "output_token_throughput": 1283.4540992269378, + "total_token_throughput": 42781.017081052414, + "total_input_tokens": 10485760, + "total_output_tokens": 324308, + "e2e_ms": { + "mean": 12341.739772960294, + "p50": 12492.597976524848, + "p90": 13279.146579967346, + "p95": 13909.506630254331, + "p99": 19709.734952092404 + }, + "ttft_ms": { + "mean": 1110.3961577793825, + "p50": 913.746756996261, + "p90": 1339.3651077931281, + "p95": 1837.394617483369, + "p99": 8464.328227938386 + }, + "tpot_ms": { + "mean": 89.2549826316773, + "p50": 91.65863783074008, + "p90": 97.26721230404054, + "p95": 99.2759227111943, + "p99": 101.99371619496534 + }, + "itl_ms": { + "mean": 89.17840043206004, + "p50": 91.65084444093665, + "p90": 97.2365819717883, + "p95": 99.26035149961592, + "p99": 101.91815370646901 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_128.jsonl" + }, + { + "name": "c128_i4096_o2048", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 299.51152995903976, + "request_throughput": 8.54725025226941, + "input_token_throughput": 35009.537033295506, + "output_token_throughput": 1841.885686589241, + "total_token_throughput": 36851.42271988475, + "total_input_tokens": 10485760, + "total_output_tokens": 551666, + "e2e_ms": { + "mean": 14467.323276237676, + "p50": 13327.958792971913, + "p90": 20516.120994987432, + "p95": 24917.877360078277, + "p99": 37076.16252239852 + }, + "ttft_ms": { + "mean": 673.6397270240786, + "p50": 404.4142944912892, + "p90": 694.2803778627421, + "p95": 903.5128011921189, + "p99": 8456.423838282935 + }, + "tpot_ms": { + "mean": 64.31002423542024, + "p50": 65.36100197871913, + "p90": 69.56556259883966, + "p95": 71.20312924578917, + "p99": 77.97612688308777 + }, + "itl_ms": { + "mean": 63.97966287460745, + "p50": 65.00690745694432, + "p90": 69.20030281423097, + "p95": 70.85190648050559, + "p99": 77.09933678045897 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_2048.jsonl" + }, + { + "name": "c128_i4096_o256", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 288.9844872049871, + "request_throughput": 8.85860699569005, + "input_token_throughput": 36284.85425434644, + "output_token_throughput": 1743.9586632292514, + "total_token_throughput": 38028.81291757569, + "total_input_tokens": 10485760, + "total_output_tokens": 503977, + "e2e_ms": { + "mean": 14052.644563852664, + "p50": 14011.578131030547, + "p90": 18207.99634809373, + "p95": 18727.13703426707, + "p99": 21844.01485296432 + }, + "ttft_ms": { + "mean": 688.4898013677685, + "p50": 420.76750652631745, + "p90": 717.7646088181062, + "p95": 1029.3209117808167, + "p99": 8474.333298587006 + }, + "tpot_ms": { + "mean": 68.27357329740917, + "p50": 69.55606692148757, + "p90": 73.2924286689247, + "p95": 74.6707608597957, + "p99": 80.07070039799193 + }, + "itl_ms": { + "mean": 67.9460994704431, + "p50": 69.2711814984261, + "p90": 72.90093740416758, + "p95": 74.33009236125588, + "p99": 79.8225049918672 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_256.jsonl" + }, + { + "name": "c128_i4096_o4096", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 297.6913547680015, + "request_throughput": 8.599510731492602, + "input_token_throughput": 35223.5959561937, + "output_token_throughput": 1820.8355443255352, + "total_token_throughput": 37044.43150051923, + "total_input_tokens": 10485760, + "total_output_tokens": 542047, + "e2e_ms": { + "mean": 14368.531745259383, + "p50": 13428.582592488965, + "p90": 20025.193709513405, + "p95": 24055.67811630026, + "p99": 33864.98227334923 + }, + "ttft_ms": { + "mean": 679.8773801880316, + "p50": 405.75079250265844, + "p90": 709.7748359898104, + "p95": 979.6798837225666, + "p99": 8471.744555426993 + }, + "tpot_ms": { + "mean": 65.1796741939593, + "p50": 66.18706630407843, + "p90": 70.68885655515605, + "p95": 72.12222651762532, + "p99": 79.0404013413206 + }, + "itl_ms": { + "mean": 64.83921205698559, + "p50": 65.83542271383942, + "p90": 70.3737927563315, + "p95": 71.8124721641636, + "p99": 78.200747942912 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_4096.jsonl" + }, + { + "name": "c128_i4096_o512", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 296.8965953789884, + "request_throughput": 8.62253067177197, + "input_token_throughput": 35317.88563157799, + "output_token_throughput": 1815.7904414896925, + "total_token_throughput": 37133.67607306768, + "total_input_tokens": 10485760, + "total_output_tokens": 539102, + "e2e_ms": { + "mean": 14377.251431213948, + "p50": 13426.22295350884, + "p90": 20687.27437831694, + "p95": 24981.060059840107, + "p99": 33674.280850780175 + }, + "ttft_ms": { + "mean": 666.1897538902394, + "p50": 402.21049898536876, + "p90": 685.1608576835132, + "p95": 888.769919294283, + "p99": 8469.317924450152 + }, + "tpot_ms": { + "mean": 65.45340909588701, + "p50": 66.52154384660442, + "p90": 70.71509014012665, + "p95": 72.23137798844691, + "p99": 78.81060099442449 + }, + "itl_ms": { + "mean": 65.12317600401387, + "p50": 66.27417706121985, + "p90": 70.3212920331301, + "p95": 71.8430878627161, + "p99": 78.63558289697215 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_512.jsonl" + }, + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 155.28525411698502, + "request_throughput": 16.485789423839368, + "input_token_throughput": 8440.724185005756, + "output_token_throughput": 3401.044117184036, + "total_token_throughput": 11841.768302189792, + "total_input_tokens": 1310720, + "total_output_tokens": 528132, + "e2e_ms": { + "mean": 7433.739403959908, + "p50": 7612.308174022473, + "p90": 9559.700762154534, + "p95": 9800.363442124217, + "p99": 10039.981560425947 + }, + "ttft_ms": { + "mean": 216.11621198128432, + "p50": 176.26684298738837, + "p90": 270.79010872403154, + "p95": 439.64523445174757, + "p99": 1351.9783651607574 + }, + "tpot_ms": { + "mean": 35.17330734203888, + "p50": 36.002418438770434, + "p90": 38.22760091307209, + "p95": 38.71466623999261, + "p99": 39.83368630610904 + }, + "itl_ms": { + "mean": 35.03133331348015, + "p50": 35.86981591799907, + "p90": 38.09707126092343, + "p95": 38.51183454514206, + "p99": 39.7125292160622 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_512_256.jsonl" + }, + { + "name": "c16_i512_o256", + "config": { + "concurrency": 16, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 60.82467504602391, + "request_throughput": 5.261022763506211, + "input_token_throughput": 2693.64365491518, + "output_token_throughput": 1089.672899195088, + "total_token_throughput": 3783.316554110268, + "total_input_tokens": 163840, + "total_output_tokens": 66279, + "e2e_ms": { + "mean": 2748.5973666503924, + "p50": 2736.172020493541, + "p90": 3588.4529045782983, + "p95": 4105.760691943578, + "p99": 4640.225247456692 + }, + "ttft_ms": { + "mean": 145.5557072244119, + "p50": 118.74175499542616, + "p90": 195.0831679045223, + "p95": 311.20013739855494, + "p99": 388.93986369890627 + }, + "tpot_ms": { + "mean": 12.620319833878325, + "p50": 12.123861945206848, + "p90": 15.690948749874153, + "p95": 16.997498412684262, + "p99": 17.834258368315975 + }, + "itl_ms": { + "mean": 12.568962422917354, + "p50": 12.075563610004078, + "p90": 15.655471615624732, + "p95": 16.959495267951805, + "p99": 17.8226512715222 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_16_512_256.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 52.340283242985606, + "request_throughput": 0.38211485992828104, + "input_token_throughput": 391.2856165665598, + "output_token_throughput": 85.13519079202102, + "total_token_throughput": 476.4208073585808, + "total_input_tokens": 20480, + "total_output_tokens": 4456, + "e2e_ms": { + "mean": 1666.6323251556605, + "p50": 1607.7605970203876, + "p90": 2214.885215985123, + "p95": 2301.1320795718348, + "p99": 2358.686553525622 + }, + "ttft_ms": { + "mean": 115.80964514578227, + "p50": 114.3495729775168, + "p90": 116.58464862266555, + "p95": 118.25840408855585, + "p99": 140.49979043367782 + }, + "tpot_ms": { + "mean": 6.992964128556314, + "p50": 6.993668697891669, + "p90": 6.99977319934427, + "p95": 7.0010958709895394, + "p99": 7.001298266553494 + }, + "itl_ms": { + "mean": 6.958230151605854, + "p50": 6.958213456912434, + "p90": 6.961470347430554, + "p95": 6.962435742448271, + "p99": 6.963362603158629 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o128", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 32.71276432002196, + "request_throughput": 0.611382144423635, + "input_token_throughput": 626.0553158898023, + "output_token_throughput": 78.07349984289819, + "total_token_throughput": 704.1288157327004, + "total_input_tokens": 20480, + "total_output_tokens": 2554, + "e2e_ms": { + "mean": 997.0761572010815, + "p50": 998.1900005077478, + "p90": 1002.62519911048, + "p95": 1004.8363747657277, + "p99": 1011.9732421101071 + }, + "ttft_ms": { + "mean": 115.304818644654, + "p50": 115.1561715232674, + "p90": 118.39750949875452, + "p95": 122.94176129507832, + "p99": 128.4698994399514 + }, + "tpot_ms": { + "mean": 6.95965548413073, + "p50": 6.958289842596131, + "p90": 6.960662914757199, + "p95": 6.970090178271585, + "p99": 7.007105844121334 + }, + "itl_ms": { + "mean": 6.956049581701196, + "p50": 6.9574823148830935, + "p90": 6.959047572280273, + "p95": 6.959772129051799, + "p99": 6.965634709180105 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_128.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 48.49656022601994, + "request_throughput": 0.41240038276507224, + "input_token_throughput": 422.29799195143397, + "output_token_throughput": 83.01619705060904, + "total_token_throughput": 505.314189002043, + "total_input_tokens": 20480, + "total_output_tokens": 4026, + "e2e_ms": { + "mean": 1516.367995305336, + "p50": 1535.2776270301547, + "p90": 1693.6148332140874, + "p95": 1700.7698216184508, + "p99": 1778.3540971053296 + }, + "ttft_ms": { + "mean": 115.62368175073061, + "p50": 115.08111699367873, + "p90": 118.04690642748028, + "p95": 119.49695778021123, + "p99": 126.28927154641131 + }, + "tpot_ms": { + "mean": 6.993729136018383, + "p50": 6.993365077923091, + "p90": 6.998930970311811, + "p95": 7.004203024148848, + "p99": 7.006087367499652 + }, + "itl_ms": { + "mean": 6.955256937332673, + "p50": 6.955609315216483, + "p90": 6.9578465340344176, + "p95": 6.958589018775167, + "p99": 6.959668077607412 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 48.48484487499809, + "request_throughput": 0.4125000307119326, + "input_token_throughput": 422.40003144901897, + "output_token_throughput": 81.26250605025072, + "total_token_throughput": 503.6625374992697, + "total_input_tokens": 20480, + "total_output_tokens": 3940, + "e2e_ms": { + "mean": 1485.3911130456254, + "p50": 1523.8948875048663, + "p90": 1818.4952258015983, + "p95": 1837.5514468731126, + "p99": 1880.363019790384 + }, + "ttft_ms": { + "mean": 114.81126984290313, + "p50": 113.8783980277367, + "p90": 117.8437611728441, + "p95": 120.57862681685947, + "p99": 125.23157255607656 + }, + "tpot_ms": { + "mean": 6.994496555731443, + "p50": 6.995160303404432, + "p90": 7.0082143222680315, + "p95": 7.010770282737416, + "p99": 7.011674420422894 + }, + "itl_ms": { + "mean": 6.955705517495981, + "p50": 6.954842998457913, + "p90": 6.95964445904527, + "p95": 6.959962120696949, + "p99": 6.963102689140168 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_256.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 51.98334731196519, + "request_throughput": 0.3847385948422088, + "input_token_throughput": 393.9723211184218, + "output_token_throughput": 82.7187978910749, + "total_token_throughput": 476.69111900949673, + "total_input_tokens": 20480, + "total_output_tokens": 4300, + "e2e_ms": { + "mean": 1612.523135956144, + "p50": 1597.956053999951, + "p90": 2029.4950069335755, + "p95": 2093.7656364869326, + "p99": 2163.9699145313352 + }, + "ttft_ms": { + "mean": 116.50667509820778, + "p50": 114.9420035071671, + "p90": 118.94925460801461, + "p95": 126.80927838664503, + "p99": 137.82952686306086 + }, + "tpot_ms": { + "mean": 6.991852548113963, + "p50": 6.98992295172064, + "p90": 7.002194919252066, + "p95": 7.004974589785324, + "p99": 7.007806217299085 + }, + "itl_ms": { + "mean": 6.954919126074468, + "p50": 6.955506687569439, + "p90": 6.958137890710187, + "p95": 6.959299433278432, + "p99": 6.959750128390998 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_4096.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 49.68123743298929, + "request_throughput": 0.4025664623788058, + "input_token_throughput": 412.2280574758971, + "output_token_throughput": 78.90302662624593, + "total_token_throughput": 491.131084102143, + "total_input_tokens": 20480, + "total_output_tokens": 3920, + "e2e_ms": { + "mean": 1497.0933472388424, + "p50": 1472.0589299686253, + "p90": 1865.7209743745627, + "p95": 2052.554446668364, + "p99": 2150.5699093313888 + }, + "ttft_ms": { + "mean": 132.68365324183833, + "p50": 115.42460750206374, + "p90": 173.59126399969682, + "p95": 174.47906527668238, + "p99": 175.38653943687677 + }, + "tpot_ms": { + "mean": 6.998256694810179, + "p50": 6.995677604377155, + "p90": 7.012659735752915, + "p95": 7.016148772127566, + "p99": 7.016167255125867 + }, + "itl_ms": { + "mean": 6.957685190034067, + "p50": 6.957637038476368, + "p90": 6.964746559207174, + "p95": 6.965498266336599, + "p99": 6.965703383228294 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_512.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 257.4472376179765, + "request_throughput": 0.07768582092800626, + "input_token_throughput": 10182.435920675636, + "output_token_throughput": 8.902795078349518, + "total_token_throughput": 10191.338715753987, + "total_input_tokens": 2621440, + "total_output_tokens": 2292, + "e2e_ms": { + "mean": 7290.28443134448, + "p50": 7286.690048524179, + "p90": 7319.372808613116, + "p95": 7408.333866103203, + "p99": 7425.827623613295 + }, + "ttft_ms": { + "mean": 6471.478171434137, + "p50": 6459.610841469839, + "p90": 6570.740136282984, + "p95": 6574.879066209542, + "p99": 6590.681217216188 + }, + "tpot_ms": { + "mean": 7.209416115326391, + "p50": 7.197772930998452, + "p90": 7.2473051953872965, + "p95": 7.251998598967419, + "p99": 7.316937038715112 + }, + "itl_ms": { + "mean": 7.187574219107276, + "p50": 7.195279056382976, + "p90": 7.206139789597164, + "p95": 7.207912252563582, + "p99": 7.21993116777318 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 259.43278859899146, + "request_throughput": 0.07709125784757397, + "input_token_throughput": 10104.505348597215, + "output_token_throughput": 13.109368396979955, + "total_token_throughput": 10117.614716994196, + "total_input_tokens": 2621440, + "total_output_tokens": 3401, + "e2e_ms": { + "mean": 7749.514106288552, + "p50": 7651.893796457443, + "p90": 8206.727583683096, + "p95": 8208.241063222522, + "p99": 8219.552055849927 + }, + "ttft_ms": { + "mean": 6523.804933592328, + "p50": 6512.4543030106, + "p90": 6598.4229369903915, + "p95": 6605.214903116575, + "p99": 6683.361206200789 + }, + "tpot_ms": { + "mean": 7.253172623286035, + "p50": 7.254124181907761, + "p90": 7.272623823075956, + "p95": 7.2848265816878275, + "p99": 7.296334035252958 + }, + "itl_ms": { + "mean": 7.179794252086232, + "p50": 7.175094242766207, + "p90": 7.214930474917503, + "p95": 7.216970495949786, + "p99": 7.221291095815641 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 281.0289259409765, + "request_throughput": 0.07116705133833992, + "input_token_throughput": 9328.00775301889, + "output_token_throughput": 15.400549909616759, + "total_token_throughput": 9343.408302928507, + "total_input_tokens": 2621440, + "total_output_tokens": 4328, + "e2e_ms": { + "mean": 8083.131315253559, + "p50": 7868.732059490867, + "p90": 9030.448771850208, + "p95": 9930.97106947098, + "p99": 10147.713469895534 + }, + "ttft_ms": { + "mean": 6520.167630907963, + "p50": 6511.3907440099865, + "p90": 6591.417226468911, + "p95": 6595.290473787463, + "p99": 6639.392100346158 + }, + "tpot_ms": { + "mean": 7.255575324965767, + "p50": 7.2530971403025575, + "p90": 7.261777192937413, + "p95": 7.272023605883151, + "p99": 7.29306066564882 + }, + "itl_ms": { + "mean": 7.187150700754481, + "p50": 7.189161557386939, + "p90": 7.219701732356514, + "p95": 7.228312448436723, + "p99": 7.247371476856312 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_131072_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 85.55190685600974, + "request_throughput": 0.2337762036521465, + "input_token_throughput": 3830.189320636768, + "output_token_throughput": 66.12359920300963, + "total_token_throughput": 3896.312919839778, + "total_input_tokens": 327680, + "total_output_tokens": 5657, + "e2e_ms": { + "mean": 2761.828099846025, + "p50": 2430.3790545091033, + "p90": 3463.2150286342962, + "p95": 4275.721653646907, + "p99": 5529.526115544721 + }, + "ttft_ms": { + "mean": 761.5903040976264, + "p50": 760.6050779868383, + "p90": 766.9433312665205, + "p95": 767.1664872235851, + "p99": 767.5652326649288 + }, + "tpot_ms": { + "mean": 7.09461820686801, + "p50": 7.094541371586382, + "p90": 7.10644475822362, + "p95": 7.10756015048735, + "p99": 7.107590855281464 + }, + "itl_ms": { + "mean": 7.060456193738931, + "p50": 7.055744335329496, + "p90": 7.081214189454361, + "p95": 7.090980342578718, + "p99": 7.095196453397732 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 54.062904359947424, + "request_throughput": 0.36993942957339576, + "input_token_throughput": 6061.087614130516, + "output_token_throughput": 46.98230755582126, + "total_token_throughput": 6108.069921686337, + "total_input_tokens": 327680, + "total_output_tokens": 2540, + "e2e_ms": { + "mean": 1645.3096443059621, + "p50": 1645.1492685009725, + "p90": 1648.5932470939588, + "p95": 1649.6672159439186, + "p99": 1651.5440808032872 + }, + "ttft_ms": { + "mean": 758.9813886006596, + "p50": 759.2984469956718, + "p90": 761.3394041894935, + "p95": 762.0397499384126, + "p99": 764.5994139945833 + }, + "tpot_ms": { + "mean": 7.03435123575637, + "p50": 7.034438234219875, + "p90": 7.0417456302998795, + "p95": 7.043358887289835, + "p99": 7.044686933047145 + }, + "itl_ms": { + "mean": 7.0338002467502125, + "p50": 7.033853519787376, + "p90": 7.041230728519001, + "p95": 7.042800428369822, + "p99": 7.043882879430414 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_128.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 90.19640143198194, + "request_throughput": 0.22173833636902035, + "input_token_throughput": 3632.9609030700294, + "output_token_throughput": 62.70760152515896, + "total_token_throughput": 3695.6685045951886, + "total_input_tokens": 327680, + "total_output_tokens": 5656, + "e2e_ms": { + "mean": 2762.5432386033935, + "p50": 2452.210796473082, + "p90": 3715.490449598292, + "p95": 3858.2562830328243, + "p99": 4063.0771798180645 + }, + "ttft_ms": { + "mean": 762.252552350401, + "p50": 762.4800890043844, + "p90": 767.4323470098898, + "p95": 769.1343927930575, + "p99": 769.4857225881424 + }, + "tpot_ms": { + "mean": 7.097202645035762, + "p50": 7.097632476244429, + "p90": 7.107065638170411, + "p95": 7.107487881600403, + "p99": 7.107727561069631 + }, + "itl_ms": { + "mean": 7.064960211576161, + "p50": 7.06291316644792, + "p90": 7.086573517755428, + "p95": 7.088723471803392, + "p99": 7.09011580964756 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_2048.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 73.12888823298272, + "request_throughput": 0.2734897313942695, + "input_token_throughput": 4480.855759163712, + "output_token_throughput": 56.18846531495267, + "total_token_throughput": 4537.044224478665, + "total_input_tokens": 327680, + "total_output_tokens": 4109, + "e2e_ms": { + "mean": 2205.644827496144, + "p50": 2216.0204490064643, + "p90": 2557.4225594173186, + "p95": 2558.677955370513, + "p99": 2568.8925430789823 + }, + "ttft_ms": { + "mean": 757.2442296514055, + "p50": 759.4909319886938, + "p90": 765.4850837308913, + "p95": 769.1775480052456, + "p99": 772.9351856047288 + }, + "tpot_ms": { + "mean": 7.086063474876053, + "p50": 7.086371160285189, + "p90": 7.106006101920233, + "p95": 7.107053158383294, + "p99": 7.108264589840503 + }, + "itl_ms": { + "mean": 7.0536245266423645, + "p50": 7.0581677382566905, + "p90": 7.074847337643169, + "p95": 7.076649383083078, + "p99": 7.076766213604275 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 75.42876927595353, + "request_throughput": 0.2651508196671047, + "input_token_throughput": 4344.231029425843, + "output_token_throughput": 53.958191802255804, + "total_token_throughput": 4398.189221228099, + "total_input_tokens": 327680, + "total_output_tokens": 4070, + "e2e_ms": { + "mean": 2198.956436300068, + "p50": 2213.818849500967, + "p90": 2626.4926128555094, + "p95": 2832.268335088156, + "p99": 2889.4597958168015 + }, + "ttft_ms": { + "mean": 761.9774985505501, + "p50": 761.5274374838918, + "p90": 764.6430697699543, + "p95": 768.2955486234277, + "p99": 778.232998540625 + }, + "tpot_ms": { + "mean": 7.0961170639464255, + "p50": 7.095928777097879, + "p90": 7.106023095899129, + "p95": 7.106170595785851, + "p99": 7.107090797877847 + }, + "itl_ms": { + "mean": 7.053604431683894, + "p50": 7.054318090364773, + "p90": 7.067077616323665, + "p95": 7.068994884135261, + "p99": 7.0774931702040575 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_512.jsonl" + }, + { + "name": "c1_i2048_o1024", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 52.1534125340404, + "request_throughput": 0.3834840143383916, + "input_token_throughput": 785.375261365026, + "output_token_throughput": 81.93135966339736, + "total_token_throughput": 867.3066210284234, + "total_input_tokens": 40960, + "total_output_tokens": 4273, + "e2e_ms": { + "mean": 1639.0007649490144, + "p50": 1520.7974360091612, + "p90": 2190.022072021384, + "p95": 2263.6719663161794, + "p99": 3129.8387580644326 + }, + "ttft_ms": { + "mean": 135.1462520484347, + "p50": 133.8202319748234, + "p90": 138.35612262482755, + "p95": 140.2339472057065, + "p99": 150.81769502896347 + }, + "tpot_ms": { + "mean": 7.0747086929852685, + "p50": 7.0748485663652305, + "p90": 7.088887935875075, + "p95": 7.091126444031059, + "p99": 7.099713838744482 + }, + "itl_ms": { + "mean": 7.03472727239529, + "p50": 7.033652576394227, + "p90": 7.045419228656695, + "p95": 7.045716443511645, + "p99": 7.0469850743595 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_1024.jsonl" + }, + { + "name": "c1_i2048_o128", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 33.24076693301322, + "request_throughput": 0.6016708351014882, + "input_token_throughput": 1232.2218702878479, + "output_token_throughput": 74.96818605364544, + "total_token_throughput": 1307.1900563414933, + "total_input_tokens": 40960, + "total_output_tokens": 2492, + "e2e_ms": { + "mean": 1006.6183711052872, + "p50": 1025.5568735010456, + "p90": 1031.097009422956, + "p95": 1053.4016749501461, + "p99": 1075.8364333986538 + }, + "ttft_ms": { + "mean": 136.94067875039764, + "p50": 133.35946449660696, + "p90": 137.41420948063026, + "p95": 159.17179067619148, + "p99": 181.960963755846 + }, + "tpot_ms": { + "mean": 7.038047122125526, + "p50": 7.034313641800216, + "p90": 7.047546141091331, + "p95": 7.10328894771225, + "p99": 7.104833651601928 + }, + "itl_ms": { + "mean": 7.029471314430009, + "p50": 7.0264248858484635, + "p90": 7.037256039242722, + "p95": 7.0381679429625, + "p99": 7.040319012296496 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_128.jsonl" + }, + { + "name": "c1_i2048_o2048", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 51.476043572009075, + "request_throughput": 0.38853024848388545, + "input_token_throughput": 795.7099488949974, + "output_token_throughput": 84.60246160736605, + "total_token_throughput": 880.3124105023635, + "total_input_tokens": 40960, + "total_output_tokens": 4355, + "e2e_ms": { + "mean": 1667.398910401971, + "p50": 1539.4108625187073, + "p90": 2376.00737574394, + "p95": 3179.9570861068787, + "p99": 3366.919990839087 + }, + "ttft_ms": { + "mean": 134.55304539820645, + "p50": 133.54374349000864, + "p90": 135.26073460816406, + "p95": 137.97220440756064, + "p99": 148.68526168575042 + }, + "tpot_ms": { + "mean": 7.0742273781296205, + "p50": 7.074927825367073, + "p90": 7.082008953453719, + "p95": 7.089882750816399, + "p99": 7.095734023629707 + }, + "itl_ms": { + "mean": 7.034275217543682, + "p50": 7.034182887314636, + "p90": 7.044180244236356, + "p95": 7.0465115520927295, + "p99": 7.051333929426694 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_2048.jsonl" + }, + { + "name": "c1_i2048_o256", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 48.447020565974526, + "request_throughput": 0.4128220841313504, + "input_token_throughput": 845.4596283010056, + "output_token_throughput": 79.01414690274046, + "total_token_throughput": 924.4737752037462, + "total_input_tokens": 40960, + "total_output_tokens": 3828, + "e2e_ms": { + "mean": 1479.6458801545668, + "p50": 1452.7736474992707, + "p90": 1930.3096152900252, + "p95": 1930.9429561020806, + "p99": 1931.7471896251664 + }, + "ttft_ms": { + "mean": 133.364660301595, + "p50": 133.06230350281112, + "p90": 134.4277158670593, + "p95": 134.8362378514139, + "p99": 138.7125235464191 + }, + "tpot_ms": { + "mean": 7.072852898068682, + "p50": 7.075266202282437, + "p90": 7.0822706008898235, + "p95": 7.0830494113516, + "p99": 7.091806624947279 + }, + "itl_ms": { + "mean": 7.0355477412239775, + "p50": 7.036148776481396, + "p90": 7.045096764934486, + "p95": 7.0460076927887645, + "p99": 7.047783690598319 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_256.jsonl" + }, + { + "name": "c1_i2048_o4096", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 52.65344714699313, + "request_throughput": 0.37984217717343005, + "input_token_throughput": 777.9167788511847, + "output_token_throughput": 77.54478046995574, + "total_token_throughput": 855.4615593211405, + "total_input_tokens": 40960, + "total_output_tokens": 4083, + "e2e_ms": { + "mean": 1571.4832365047187, + "p50": 1578.1495150004048, + "p90": 1986.1723132315092, + "p95": 2033.6844808247406, + "p99": 2095.4177425953094 + }, + "ttft_ms": { + "mean": 134.26353290269617, + "p50": 133.42173700220883, + "p90": 134.74032204248942, + "p95": 137.78332775691524, + "p99": 149.1545207169838 + }, + "tpot_ms": { + "mean": 7.075507301509984, + "p50": 7.0742508989005675, + "p90": 7.083799128494104, + "p95": 7.085761582440888, + "p99": 7.097104248244071 + }, + "itl_ms": { + "mean": 7.036068429931236, + "p50": 7.036433208274854, + "p90": 7.044877956843721, + "p95": 7.045143621820257, + "p99": 7.0458232713827575 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_4096.jsonl" + }, + { + "name": "c1_i2048_o512", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 50.388948728039395, + "request_throughput": 0.3969124283172595, + "input_token_throughput": 812.8766531937474, + "output_token_throughput": 79.10464696362982, + "total_token_throughput": 891.9813001573773, + "total_input_tokens": 40960, + "total_output_tokens": 3986, + "e2e_ms": { + "mean": 1537.0369591546478, + "p50": 1504.8605250194669, + "p90": 2335.837259324035, + "p95": 2769.402894700761, + "p99": 2894.6374381514033 + }, + "ttft_ms": { + "mean": 133.85857905086596, + "p50": 133.14539249404334, + "p90": 134.4122292357497, + "p95": 135.40067606954835, + "p99": 147.36564562539567 + }, + "tpot_ms": { + "mean": 7.080310007154715, + "p50": 7.080751560016797, + "p90": 7.098766246633662, + "p95": 7.102012714910587, + "p99": 7.103908377730314 + }, + "itl_ms": { + "mean": 7.035449573803811, + "p50": 7.035650407287871, + "p90": 7.044265468773851, + "p95": 7.046153439982385, + "p99": 7.049547889021412 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_512.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 556.8975440699724, + "request_throughput": 0.03591324869891519, + "input_token_throughput": 9414.442666928424, + "output_token_throughput": 6.2722488852655385, + "total_token_throughput": 9420.714915813689, + "total_input_tokens": 5242880, + "total_output_tokens": 3493, + "e2e_ms": { + "mean": 16530.717764247674, + "p50": 16664.499283506302, + "p90": 17023.02305897465, + "p95": 17025.054730603006, + "p99": 17032.912425303366 + }, + "ttft_ms": { + "mean": 15234.609863755759, + "p50": 15248.796680971282, + "p90": 15315.202502999455, + "p95": 15316.530187413446, + "p99": 15320.265578298713 + }, + "tpot_ms": { + "mean": 7.4726540033093825, + "p50": 7.4731072332071005, + "p90": 7.507545618103787, + "p95": 7.547523731020181, + "p99": 7.567509443686337 + }, + "itl_ms": { + "mean": 7.3375862011256014, + "p50": 7.328241241606573, + "p90": 7.390136176649624, + "p95": 7.407534864068883, + "p99": 7.539143762656007 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_262144_256.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 50.77059198997449, + "request_throughput": 0.3939288319495927, + "input_token_throughput": 1613.5324956655318, + "output_token_throughput": 78.27365890838408, + "total_token_throughput": 1691.8061545739158, + "total_input_tokens": 81920, + "total_output_tokens": 3974, + "e2e_ms": { + "mean": 1622.7367079496616, + "p50": 1548.5019845073111, + "p90": 1970.093786006328, + "p95": 2090.628466138151, + "p99": 2641.6305804229332 + }, + "ttft_ms": { + "mean": 223.9771443477366, + "p50": 223.59500601305626, + "p90": 227.59674831759185, + "p95": 228.83622981898952, + "p99": 229.9823219521204 + }, + "tpot_ms": { + "mean": 7.076595673752794, + "p50": 7.081092141277011, + "p90": 7.086373823695292, + "p95": 7.087853686347765, + "p99": 7.090011781365773 + }, + "itl_ms": { + "mean": 7.034033030946358, + "p50": 7.037121284554964, + "p90": 7.044552039641219, + "p95": 7.0486115542576515, + "p99": 7.053144998174831 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 36.183199760038406, + "request_throughput": 0.5527427129893714, + "input_token_throughput": 2264.034152404465, + "output_token_throughput": 70.1706874140007, + "total_token_throughput": 2334.204839818466, + "total_input_tokens": 81920, + "total_output_tokens": 2539, + "e2e_ms": { + "mean": 1108.6190134024946, + "p50": 1115.3957745118532, + "p90": 1117.3203395039309, + "p95": 1117.8947942622472, + "p99": 1119.9307620862965 + }, + "ttft_ms": { + "mean": 223.36841770156752, + "p50": 223.17008301615715, + "p90": 224.61351838428527, + "p95": 225.7140525529394, + "p99": 227.20984732557554 + }, + "tpot_ms": { + "mean": 7.029005154512127, + "p50": 7.0305118150734085, + "p90": 7.034080769586208, + "p95": 7.03638333245587, + "p99": 7.070559654767858 + }, + "itl_ms": { + "mean": 7.02481579687929, + "p50": 7.025486673228443, + "p90": 7.033457389691713, + "p95": 7.033528619208447, + "p99": 7.033584231027681 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_128.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 54.08545239997329, + "request_throughput": 0.3697852030911343, + "input_token_throughput": 1514.640191861286, + "output_token_throughput": 78.3389952748568, + "total_token_throughput": 1592.9791871361429, + "total_input_tokens": 81920, + "total_output_tokens": 4237, + "e2e_ms": { + "mean": 1715.1450212550117, + "p50": 1574.5812459790614, + "p90": 2619.9436117953155, + "p95": 2736.4090675429907, + "p99": 2965.236396695254 + }, + "ttft_ms": { + "mean": 223.30448884749785, + "p50": 223.00233450368978, + "p90": 224.80262753088027, + "p95": 226.85476079350337, + "p99": 227.96985933324322 + }, + "tpot_ms": { + "mean": 7.075842410128919, + "p50": 7.075092399596053, + "p90": 7.085437742444061, + "p95": 7.0869074893536945, + "p99": 7.087335458129968 + }, + "itl_ms": { + "mean": 7.034790197294845, + "p50": 7.036254612914846, + "p90": 7.052687796748768, + "p95": 7.054119502062382, + "p99": 7.054655302466439 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 53.93014029797632, + "request_throughput": 0.37085013852171417, + "input_token_throughput": 1519.0021673849412, + "output_token_throughput": 77.17391382636872, + "total_token_throughput": 1596.17608121131, + "total_input_tokens": 81920, + "total_output_tokens": 4162, + "e2e_ms": { + "mean": 1687.853766803164, + "p50": 1731.4121435047127, + "p90": 2022.2392167255748, + "p95": 2022.9304053325905, + "p99": 2023.168032289832 + }, + "ttft_ms": { + "mean": 224.11002360167913, + "p50": 223.77661100472324, + "p90": 225.24597371229902, + "p95": 227.58357299608178, + "p99": 229.3452226149384 + }, + "tpot_ms": { + "mean": 7.069597663492597, + "p50": 7.071157336921616, + "p90": 7.084982468067765, + "p95": 7.0854949991775005, + "p99": 7.086657547444251 + }, + "itl_ms": { + "mean": 7.036856933328044, + "p50": 7.036261300358248, + "p90": 7.0516792783632365, + "p95": 7.053535119733116, + "p99": 7.053621047452621 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_256.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 50.60050758899888, + "request_throughput": 0.39525295205434313, + "input_token_throughput": 1618.9560916145895, + "output_token_throughput": 72.98345759683446, + "total_token_throughput": 1691.9395492114238, + "total_input_tokens": 81920, + "total_output_tokens": 3693, + "e2e_ms": { + "mean": 1522.8591587103438, + "p50": 1464.6899780200329, + "p90": 1853.4205963020217, + "p95": 2146.742542926222, + "p99": 2282.4891966208816 + }, + "ttft_ms": { + "mean": 223.4261866658926, + "p50": 223.1349789944943, + "p90": 227.13108609896153, + "p95": 228.4300401690416, + "p99": 229.7299312404357 + }, + "tpot_ms": { + "mean": 7.07648349021855, + "p50": 7.077039959071561, + "p90": 7.086898134567167, + "p95": 7.090804919410713, + "p99": 7.091896303988687 + }, + "itl_ms": { + "mean": 7.031315753257085, + "p50": 7.035521328040964, + "p90": 7.0425933163626375, + "p95": 7.043398734457423, + "p99": 7.044492975297885 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_4096.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 59.7474733509589, + "request_throughput": 0.3347421887201698, + "input_token_throughput": 1371.1040049978155, + "output_token_throughput": 81.14150654576916, + "total_token_throughput": 1452.2455115435846, + "total_input_tokens": 81920, + "total_output_tokens": 4848, + "e2e_ms": { + "mean": 1931.5284742042422, + "p50": 1740.4890735051595, + "p90": 2566.8827370624062, + "p95": 2977.276942195022, + "p99": 3648.358510847319 + }, + "ttft_ms": { + "mean": 224.35228339745663, + "p50": 223.72989598079585, + "p90": 228.29745516064577, + "p95": 229.06503619451541, + "p99": 230.83653841691557 + }, + "tpot_ms": { + "mean": 7.072256846460429, + "p50": 7.074458655208373, + "p90": 7.080548995162574, + "p95": 7.081527911139119, + "p99": 7.081865160556928 + }, + "itl_ms": { + "mean": 7.03638093066817, + "p50": 7.042975338492516, + "p90": 7.048521073698659, + "p95": 7.052580513075507, + "p99": 7.055588117189501 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 53.26067367795622, + "request_throughput": 0.3755115851694098, + "input_token_throughput": 192.26193160673782, + "output_token_throughput": 80.81009312845698, + "total_token_throughput": 273.0720247351948, + "total_input_tokens": 10240, + "total_output_tokens": 4304, + "e2e_ms": { + "mean": 1663.9683443499962, + "p50": 1712.9185860103462, + "p90": 1949.9001824238803, + "p95": 1951.3943302445114, + "p99": 1953.2585076522082 + }, + "ttft_ms": { + "mean": 167.28168494882993, + "p50": 171.281192015158, + "p90": 185.20025337347764, + "p95": 203.13406442292035, + "p99": 205.33247768878937 + }, + "tpot_ms": { + "mean": 6.990167993750641, + "p50": 6.991353931776988, + "p90": 7.007845179362607, + "p95": 7.016671783989656, + "p99": 7.025201123019247 + }, + "itl_ms": { + "mean": 6.959514433374882, + "p50": 6.960831024201191, + "p90": 6.963961682596841, + "p95": 6.965071484525725, + "p99": 6.965092276522976 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_512_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 1513.7487565820338, + "request_throughput": 0.013212232157441346, + "input_token_throughput": 6927.014773360608, + "output_token_throughput": 1.2042949611507787, + "total_token_throughput": 6928.21906832176, + "total_input_tokens": 10485760, + "total_output_tokens": 1823, + "e2e_ms": { + "mean": 41303.5668047989, + "p50": 41123.62798998947, + "p90": 41813.11989680398, + "p95": 41857.1446898015, + "p99": 41958.446165995556 + }, + "ttft_ms": { + "mean": 40602.843232947635, + "p50": 40378.582418488804, + "p90": 41078.4518414177, + "p95": 41121.70196585939, + "p99": 41219.488381977426 + }, + "tpot_ms": { + "mean": 7.778498289940974, + "p50": 7.751132273666659, + "p90": 7.8827576197313025, + "p95": 7.922383648177905, + "p99": 7.92777625971768 + }, + "itl_ms": { + "mean": 7.716799590558232, + "p50": 7.735913531474868, + "p90": 7.867651437648979, + "p95": 7.879696678362727, + "p99": 7.912987946224729 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_524288_128.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 152.80665369099006, + "request_throughput": 0.13088435298403017, + "input_token_throughput": 8577.636957161401, + "output_token_throughput": 26.28157807919326, + "total_token_throughput": 8603.918535240595, + "total_input_tokens": 1310720, + "total_output_tokens": 4016, + "e2e_ms": { + "mean": 4418.26631075819, + "p50": 4360.3685425186995, + "p90": 4756.268920592266, + "p95": 5282.474650835502, + "p99": 5593.151110982871 + }, + "ttft_ms": { + "mean": 2984.4894192006905, + "p50": 2984.1709710017312, + "p90": 2992.8966829786077, + "p95": 2994.563281320734, + "p99": 3006.297108255094 + }, + "tpot_ms": { + "mean": 7.17527447116709, + "p50": 7.174407352670137, + "p90": 7.1827685568605455, + "p95": 7.184976731507607, + "p99": 7.186551507442626 + }, + "itl_ms": { + "mean": 7.12250114574196, + "p50": 7.123122260997471, + "p90": 7.143086916873816, + "p95": 7.151460329389299, + "p99": 7.155490767844952 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_1024.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 134.86410969495773, + "request_throughput": 0.14829742357130435, + "input_token_throughput": 9718.819951169002, + "output_token_throughput": 17.89208415387787, + "total_token_throughput": 9736.71203532288, + "total_input_tokens": 1310720, + "total_output_tokens": 2413, + "e2e_ms": { + "mean": 3847.8309919009916, + "p50": 3835.5290399922524, + "p90": 3901.8099384149536, + "p95": 3911.314021665021, + "p99": 3932.2561667038826 + }, + "ttft_ms": { + "mean": 2996.6540380526567, + "p50": 2983.167622500332, + "p90": 3048.6084129835945, + "p95": 3058.3009802619927, + "p99": 3078.256532016676 + }, + "tpot_ms": { + "mean": 7.114025330224445, + "p50": 7.111448429107744, + "p90": 7.122448258696979, + "p95": 7.1562583283718695, + "p99": 7.1600122270900055 + }, + "itl_ms": { + "mean": 7.109220088670858, + "p50": 7.109864420878391, + "p90": 7.118059088340184, + "p95": 7.120027479602981, + "p99": 7.148371996643857 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 153.55778337997617, + "request_throughput": 0.13024413064436033, + "input_token_throughput": 8535.679345908798, + "output_token_throughput": 25.964167443953233, + "total_token_throughput": 8561.643513352752, + "total_input_tokens": 1310720, + "total_output_tokens": 3987, + "e2e_ms": { + "mean": 4407.236540847225, + "p50": 4463.37615000084, + "p90": 4759.197858988773, + "p95": 4762.50999016338, + "p99": 4778.89939242159 + }, + "ttft_ms": { + "mean": 2986.1355356988497, + "p50": 2985.1215340022463, + "p90": 2992.1448093722574, + "p95": 2994.618826545775, + "p99": 3008.904767688364 + }, + "tpot_ms": { + "mean": 7.16319049732031, + "p50": 7.168127065689811, + "p90": 7.181539436645955, + "p95": 7.1846426181983665, + "p99": 7.210727464676715 + }, + "itl_ms": { + "mean": 7.109022871854964, + "p50": 7.132144706104665, + "p90": 7.1512719032104926, + "p95": 7.154648859390686, + "p99": 7.159991177650125 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 157.15850195498206, + "request_throughput": 0.12726005752923877, + "input_token_throughput": 8340.115130236192, + "output_token_throughput": 27.70451452411528, + "total_token_throughput": 8367.819644760308, + "total_input_tokens": 1310720, + "total_output_tokens": 4354, + "e2e_ms": { + "mean": 4541.314341998077, + "p50": 4300.338625005679, + "p90": 5846.8827306700405, + "p95": 6107.6582527923165, + "p99": 6492.713085743016 + }, + "ttft_ms": { + "mean": 2986.0649044945603, + "p50": 2985.287106508622, + "p90": 2993.119363480946, + "p95": 2996.3409428397426, + "p99": 3023.154667772469 + }, + "tpot_ms": { + "mean": 7.175812514737212, + "p50": 7.174307613181792, + "p90": 7.18507248592225, + "p95": 7.18625519569333, + "p99": 7.200756394222195 + }, + "itl_ms": { + "mean": 7.1223437938975085, + "p50": 7.12593037796584, + "p90": 7.158473757668606, + "p95": 7.163340571236723, + "p99": 7.170335645946346 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_512.jsonl" + }, + { + "name": "c2_i512_o256", + "config": { + "concurrency": 2, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 44.93843933503376, + "request_throughput": 0.8901065678268052, + "input_token_throughput": 455.73456272732426, + "output_token_throughput": 188.3242970879563, + "total_token_throughput": 644.0588598152806, + "total_input_tokens": 20480, + "total_output_tokens": 8463, + "e2e_ms": { + "mean": 1611.5773893747246, + "p50": 1769.6556154696736, + "p90": 1976.518541405676, + "p95": 2020.206581390812, + "p99": 2050.242396299145 + }, + "ttft_ms": { + "mean": 123.41228122531902, + "p50": 109.16157849715091, + "p90": 174.43694929243065, + "p95": 177.91075359564272, + "p99": 187.25801390013658 + }, + "tpot_ms": { + "mean": 7.0588251237143655, + "p50": 6.9910976638514235, + "p90": 7.33967459413643, + "p95": 7.48310221176826, + "p99": 7.683450246604985 + }, + "itl_ms": { + "mean": 7.0306507297696665, + "p50": 6.953251495132388, + "p90": 7.339486186050207, + "p95": 7.452848123203854, + "p99": 7.663379109705798 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_2_512_256.jsonl" + }, + { + "name": "c32_i1024_o1024", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 78.92528866801877, + "request_throughput": 8.108934548114409, + "input_token_throughput": 8303.548977269154, + "output_token_throughput": 1684.2003652228998, + "total_token_throughput": 9987.749342492054, + "total_input_tokens": 655360, + "total_output_tokens": 132926, + "e2e_ms": { + "mean": 3611.2669636257124, + "p50": 3619.234718993539, + "p90": 4621.976903395262, + "p95": 4919.6922949748105, + "p99": 5510.542936187703 + }, + "ttft_ms": { + "mean": 154.9310551518829, + "p50": 124.08699502702802, + "p90": 187.1325402869843, + "p95": 217.27178525179625, + "p99": 815.3002050484065 + }, + "tpot_ms": { + "mean": 16.71025031725905, + "p50": 16.85418865653697, + "p90": 18.232228898781912, + "p95": 18.607247767835663, + "p99": 18.94468969722329 + }, + "itl_ms": { + "mean": 16.631025661297535, + "p50": 16.77956095549478, + "p90": 18.166693189224386, + "p95": 18.49548590061078, + "p99": 18.88842907765743 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_1024.jsonl" + }, + { + "name": "c32_i1024_o128", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 46.467179456027225, + "request_throughput": 13.773162208083754, + "input_token_throughput": 14103.718101077764, + "output_token_throughput": 1762.90020093687, + "total_token_throughput": 15866.618302014635, + "total_input_tokens": 655360, + "total_output_tokens": 81917, + "e2e_ms": { + "mean": 2110.8549089421103, + "p50": 2038.8367799750995, + "p90": 2352.120074670529, + "p95": 2431.166996533284, + "p99": 2650.418162163114 + }, + "ttft_ms": { + "mean": 408.25481356669115, + "p50": 411.61647700937465, + "p90": 687.9995263705496, + "p95": 714.2438566574128, + "p99": 757.9774171666941 + }, + "tpot_ms": { + "mean": 13.406730838620868, + "p50": 13.020844838589312, + "p90": 16.672213566187022, + "p95": 16.994947842808116, + "p99": 17.830467695905643 + }, + "itl_ms": { + "mean": 13.40621911013612, + "p50": 13.020491358131082, + "p90": 16.67184409053857, + "p95": 16.994598978927808, + "p99": 17.830109219928456 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_128.jsonl" + }, + { + "name": "c32_i1024_o2048", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 80.99940589198377, + "request_throughput": 7.901292521249697, + "input_token_throughput": 8090.92354175969, + "output_token_throughput": 1622.2094292299496, + "total_token_throughput": 9713.13297098964, + "total_input_tokens": 655360, + "total_output_tokens": 131398, + "e2e_ms": { + "mean": 3730.4086324381387, + "p50": 3610.2917045063805, + "p90": 4984.9803586083, + "p95": 5542.758034938015, + "p99": 6244.9522893788535 + }, + "ttft_ms": { + "mean": 162.95093974731572, + "p50": 125.41421098285355, + "p90": 200.1870819774922, + "p95": 334.3690510489973, + "p99": 753.3516078814864 + }, + "tpot_ms": { + "mean": 17.450724473652407, + "p50": 16.894134123043607, + "p90": 21.074347952556618, + "p95": 23.66249520612531, + "p99": 25.289011038658256 + }, + "itl_ms": { + "mean": 17.354704705635633, + "p50": 16.8038549883253, + "p90": 21.035786438586687, + "p95": 23.611833658857822, + "p99": 25.230846912060745 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_2048.jsonl" + }, + { + "name": "c32_i1024_o256", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 76.78547257801984, + "request_throughput": 8.334909957735972, + "input_token_throughput": 8534.947796721635, + "output_token_throughput": 1672.1131704899258, + "total_token_throughput": 10207.060967211562, + "total_input_tokens": 655360, + "total_output_tokens": 128394, + "e2e_ms": { + "mean": 3544.9825747800787, + "p50": 3542.4157789966557, + "p90": 4553.223528020317, + "p95": 4746.499084433889, + "p99": 4887.241407678812 + }, + "ttft_ms": { + "mean": 154.72375358949648, + "p50": 124.83429000712931, + "p90": 188.23812043992803, + "p95": 212.0150006754553, + "p99": 753.6206560855499 + }, + "tpot_ms": { + "mean": 16.974057864604074, + "p50": 16.998088166373815, + "p90": 18.762897855757615, + "p95": 19.11395051412219, + "p99": 20.002592625161597 + }, + "itl_ms": { + "mean": 16.892495206732864, + "p50": 16.91040098053608, + "p90": 18.651316667795527, + "p95": 19.051608312570696, + "p99": 19.9012634039451 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_256.jsonl" + }, + { + "name": "c32_i1024_o4096", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 77.84559064498171, + "request_throughput": 8.22140335370758, + "input_token_throughput": 8418.717034196561, + "output_token_throughput": 1664.7827953548242, + "total_token_throughput": 10083.499829551385, + "total_input_tokens": 655360, + "total_output_tokens": 129596, + "e2e_ms": { + "mean": 3549.6161134195972, + "p50": 3534.699954005191, + "p90": 4596.044332935708, + "p95": 4954.169827623991, + "p99": 5479.05965938 + }, + "ttft_ms": { + "mean": 154.27825091619525, + "p50": 124.07905000145547, + "p90": 187.61730520636777, + "p95": 214.18526259658387, + "p99": 756.5299757034518 + }, + "tpot_ms": { + "mean": 16.84736228836508, + "p50": 16.95200728817329, + "p90": 18.559453546984084, + "p95": 18.819003744342538, + "p99": 19.489634361999443 + }, + "itl_ms": { + "mean": 16.756890267590723, + "p50": 16.850854466492688, + "p90": 18.494411360209956, + "p95": 18.739564808078377, + "p99": 19.43113616649967 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_4096.jsonl" + }, + { + "name": "c32_i1024_o512", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 77.73065714200493, + "request_throughput": 8.233559621537664, + "input_token_throughput": 8431.165052454568, + "output_token_throughput": 1700.5130904595178, + "total_token_throughput": 10131.678142914085, + "total_input_tokens": 655360, + "total_output_tokens": 132182, + "e2e_ms": { + "mean": 3574.121013048716, + "p50": 3592.20840150374, + "p90": 4519.8065511882305, + "p95": 4831.873196238303, + "p99": 5478.8229196867915 + }, + "ttft_ms": { + "mean": 155.16665441473378, + "p50": 124.03260401333682, + "p90": 194.21878742869012, + "p95": 216.89559183141677, + "p99": 751.8507089995546 + }, + "tpot_ms": { + "mean": 16.619718364896162, + "p50": 16.83363489900625, + "p90": 17.92848697055871, + "p95": 18.255712662708305, + "p99": 18.667389454301812 + }, + "itl_ms": { + "mean": 16.531331980245334, + "p50": 16.762246546481126, + "p90": 17.831700172261954, + "p95": 18.077975309344417, + "p99": 18.589417282906002 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_512.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 277.9606838519685, + "request_throughput": 2.3024839021508527, + "input_token_throughput": 37723.89625283957, + "output_token_throughput": 563.7559881794422, + "total_token_throughput": 38287.652241019015, + "total_input_tokens": 10485760, + "total_output_tokens": 156702, + "e2e_ms": { + "mean": 13073.635097725128, + "p50": 11227.634843002306, + "p90": 21339.115100027993, + "p95": 26643.75307828013, + "p99": 35597.739919722095 + }, + "ttft_ms": { + "mean": 1266.2669496049602, + "p50": 1001.5671220025979, + "p90": 1585.4935965267941, + "p95": 1984.2542961676368, + "p99": 8777.01818966947 + }, + "tpot_ms": { + "mean": 48.754423819978236, + "p50": 49.38631691431912, + "p90": 57.081412245865614, + "p95": 60.639865760342666, + "p99": 65.84542761235676 + }, + "itl_ms": { + "mean": 48.5449412439343, + "p50": 49.1246242388455, + "p90": 56.97187665833761, + "p95": 60.38546719900048, + "p99": 65.76520693153428 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 240.93721615005052, + "request_throughput": 2.6562936611728003, + "input_token_throughput": 43520.71534465516, + "output_token_throughput": 338.9015665771935, + "total_token_throughput": 43859.61691123235, + "total_input_tokens": 10485760, + "total_output_tokens": 81654, + "e2e_ms": { + "mean": 11539.012484155683, + "p50": 11515.057524986332, + "p90": 12479.302866500802, + "p95": 13150.691687557264, + "p99": 16925.014838652576 + }, + "ttft_ms": { + "mean": 3738.207139007227, + "p50": 4188.889887009282, + "p90": 4799.360683088889, + "p95": 4848.253755184123, + "p99": 8794.30169423169 + }, + "tpot_ms": { + "mean": 61.61473982648905, + "p50": 60.693794508032234, + "p90": 77.17945939071997, + "p95": 81.93347362540644, + "p99": 82.76267311701956 + }, + "itl_ms": { + "mean": 61.610156660920815, + "p50": 60.69343296038578, + "p90": 77.17903028975614, + "p95": 81.93312264109437, + "p99": 82.7623171365364 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_128.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 277.4810777360108, + "request_throughput": 2.3064635802261138, + "input_token_throughput": 37789.09929842465, + "output_token_throughput": 576.9366376481537, + "total_token_throughput": 38366.0359360728, + "total_input_tokens": 10485760, + "total_output_tokens": 160089, + "e2e_ms": { + "mean": 13129.881116174965, + "p50": 11338.306193007156, + "p90": 21268.143240688376, + "p95": 24681.926308575195, + "p99": 38211.25337462759 + }, + "ttft_ms": { + "mean": 1268.9881122932093, + "p50": 1007.1230739995372, + "p90": 1518.6964344407902, + "p95": 2250.822956187766, + "p99": 8796.36670469772 + }, + "tpot_ms": { + "mean": 47.56570146384879, + "p50": 48.1970007282515, + "p90": 56.213821805788776, + "p95": 58.27852197460004, + "p99": 64.82869783185714 + }, + "itl_ms": { + "mean": 47.31921304344923, + "p50": 48.01725076262202, + "p90": 55.902028920966025, + "p95": 57.776933802019016, + "p99": 64.19539631448289 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 264.69584388204385, + "request_throughput": 2.4178694709131987, + "input_token_throughput": 39614.37341144185, + "output_token_throughput": 508.6630678644125, + "total_token_throughput": 40123.03647930626, + "total_input_tokens": 10485760, + "total_output_tokens": 134641, + "e2e_ms": { + "mean": 12624.999633311927, + "p50": 12678.140514501138, + "p90": 15806.112510379171, + "p95": 16418.681864690734, + "p99": 19908.81419315061 + }, + "ttft_ms": { + "mean": 1298.0343049251132, + "p50": 1084.8720205249265, + "p90": 1652.756949950708, + "p95": 1994.6635739383053, + "p99": 8761.715852337074 + }, + "tpot_ms": { + "mean": 54.06142052439333, + "p50": 54.97810719405666, + "p90": 62.148850621784206, + "p95": 64.54428565345222, + "p99": 70.66540055128465 + }, + "itl_ms": { + "mean": 53.84442161161199, + "p50": 54.805519818907335, + "p90": 62.00671392397239, + "p95": 64.18330302536899, + "p99": 68.9827079291856 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_256.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 275.93009687698213, + "request_throughput": 2.3194280263139655, + "input_token_throughput": 38001.50878312801, + "output_token_throughput": 561.1384975848795, + "total_token_throughput": 38562.647280712896, + "total_input_tokens": 10485760, + "total_output_tokens": 154835, + "e2e_ms": { + "mean": 13034.17151498952, + "p50": 11792.888013500487, + "p90": 20832.816231111065, + "p95": 25370.34360906109, + "p99": 27540.55410957255 + }, + "ttft_ms": { + "mean": 1268.6738726532894, + "p50": 1003.3367375144735, + "p90": 1603.3396552025806, + "p95": 1990.857944494928, + "p99": 8787.033058899106 + }, + "tpot_ms": { + "mean": 48.88744400726067, + "p50": 49.624181990480324, + "p90": 57.32051524431882, + "p95": 60.153738932365926, + "p99": 66.01202965773221 + }, + "itl_ms": { + "mean": 48.621612374564776, + "p50": 49.502786324277764, + "p90": 57.18351559129017, + "p95": 59.71803436088074, + "p99": 65.83749939502626 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_512.jsonl" + }, + { + "name": "c32_i2048_o1024", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 87.94604192103725, + "request_throughput": 7.277189354065836, + "input_token_throughput": 14903.683797126832, + "output_token_throughput": 1574.8519998700413, + "total_token_throughput": 16478.53579699687, + "total_input_tokens": 1310720, + "total_output_tokens": 138502, + "e2e_ms": { + "mean": 4023.455450163783, + "p50": 3788.9970429823734, + "p90": 5616.974991484312, + "p95": 6506.863309702021, + "p99": 8623.713361859554 + }, + "ttft_ms": { + "mean": 200.89388552732999, + "p50": 151.6667715040967, + "p90": 236.44375635194592, + "p95": 270.092881628077, + "p99": 1352.101640147157 + }, + "tpot_ms": { + "mean": 17.77137965460594, + "p50": 17.871603376209883, + "p90": 19.549240949504515, + "p95": 19.912648025465035, + "p99": 20.514297965166563 + }, + "itl_ms": { + "mean": 17.680424030826366, + "p50": 17.754758553165367, + "p90": 19.45759664143349, + "p95": 19.809925328626218, + "p99": 20.429382566472793 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_1024.jsonl" + }, + { + "name": "c32_i2048_o128", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 59.23938118299702, + "request_throughput": 10.803623995040883, + "input_token_throughput": 22125.82194184373, + "output_token_throughput": 1375.94279974341, + "total_token_throughput": 23501.76474158714, + "total_input_tokens": 1310720, + "total_output_tokens": 81510, + "e2e_ms": { + "mean": 2732.466652070889, + "p50": 2755.088367994176, + "p90": 3012.18083424028, + "p95": 3094.9215329048457, + "p99": 3404.7346471465426 + }, + "ttft_ms": { + "mean": 367.5306692531194, + "p50": 363.3325230330229, + "p90": 541.3078215962743, + "p95": 672.5567041052273, + "p99": 1352.0049781136913 + }, + "tpot_ms": { + "mean": 18.711008783973277, + "p50": 19.23976635411895, + "p90": 20.943899588904678, + "p95": 21.551382261961805, + "p99": 22.092666742765992 + }, + "itl_ms": { + "mean": 18.69443736023786, + "p50": 19.23653134626087, + "p90": 20.932656177273753, + "p95": 21.551018130739116, + "p99": 22.078695479467857 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_128.jsonl" + }, + { + "name": "c32_i2048_o2048", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 89.13132148201112, + "request_throughput": 7.180416371692275, + "input_token_throughput": 14705.49272922578, + "output_token_throughput": 1556.097202350943, + "total_token_throughput": 16261.589931576722, + "total_input_tokens": 1310720, + "total_output_tokens": 138697, + "e2e_ms": { + "mean": 4056.525911160861, + "p50": 3742.62464849744, + "p90": 5721.507992275293, + "p95": 6825.735677420744, + "p99": 9339.784606644538 + }, + "ttft_ms": { + "mean": 199.91363017315962, + "p50": 151.93368299514987, + "p90": 230.94767580041662, + "p95": 291.09600226802223, + "p99": 1361.131886652438 + }, + "tpot_ms": { + "mean": 17.872137121852198, + "p50": 17.839739867944665, + "p90": 20.02253391370516, + "p95": 20.276428343314567, + "p99": 20.978136194178266 + }, + "itl_ms": { + "mean": 17.781535712600615, + "p50": 17.74159216290751, + "p90": 19.928170335933803, + "p95": 20.192582616625344, + "p99": 20.88261418152327 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_2048.jsonl" + }, + { + "name": "c32_i2048_o256", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 83.4020835319534, + "request_throughput": 7.673669204616456, + "input_token_throughput": 15715.674531054501, + "output_token_throughput": 1544.122095590814, + "total_token_throughput": 17259.796626645315, + "total_input_tokens": 1310720, + "total_output_tokens": 128783, + "e2e_ms": { + "mean": 3838.3926361287195, + "p50": 3821.959828987019, + "p90": 4955.226511094952, + "p95": 5110.333283661748, + "p99": 5339.235162911937 + }, + "ttft_ms": { + "mean": 202.6343736236413, + "p50": 151.54230897314847, + "p90": 241.23692908906378, + "p95": 380.56901601084917, + "p99": 1355.983304544934 + }, + "tpot_ms": { + "mean": 18.190302456605295, + "p50": 18.424039937799243, + "p90": 19.81137767760755, + "p95": 20.183518817431427, + "p99": 20.729456373731534 + }, + "itl_ms": { + "mean": 18.11254713064598, + "p50": 18.35936738325809, + "p90": 19.743566255628185, + "p95": 20.059024347520136, + "p99": 20.662909090368444 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_256.jsonl" + }, + { + "name": "c32_i2048_o4096", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 88.69394837901928, + "request_throughput": 7.215824886553289, + "input_token_throughput": 14778.009367661136, + "output_token_throughput": 1520.4532266814747, + "total_token_throughput": 16298.462594342609, + "total_input_tokens": 1310720, + "total_output_tokens": 134855, + "e2e_ms": { + "mean": 3982.980794719242, + "p50": 3718.501230527181, + "p90": 5520.199165504891, + "p95": 6383.295495348279, + "p99": 9405.390494499596 + }, + "ttft_ms": { + "mean": 199.18991788063067, + "p50": 151.69027147931047, + "p90": 225.76481199939738, + "p95": 288.1128885026552, + "p99": 1360.760756857344 + }, + "tpot_ms": { + "mean": 18.058210034347876, + "p50": 18.061947073373684, + "p90": 19.997909734484665, + "p95": 20.40678994044308, + "p99": 20.835222154193175 + }, + "itl_ms": { + "mean": 17.954757087634338, + "p50": 17.966942533526677, + "p90": 19.907732744673698, + "p95": 20.27250962252495, + "p99": 20.742511472732147 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_4096.jsonl" + }, + { + "name": "c32_i2048_o512", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 90.84299184399424, + "request_throughput": 7.045122436071675, + "input_token_throughput": 14428.41074907479, + "output_token_throughput": 1530.3216811566379, + "total_token_throughput": 15958.732430231428, + "total_input_tokens": 1310720, + "total_output_tokens": 139019, + "e2e_ms": { + "mean": 4060.401959133651, + "p50": 3761.1486914684065, + "p90": 5925.088173424592, + "p95": 6700.475931522667, + "p99": 9080.734873839541 + }, + "ttft_ms": { + "mean": 197.1673974629084, + "p50": 151.31841399124824, + "p90": 224.7515225026291, + "p95": 253.29808431270044, + "p99": 1349.4623282918474 + }, + "tpot_ms": { + "mean": 17.877858826093764, + "p50": 17.956536891848835, + "p90": 19.937387328063277, + "p95": 20.253823827590228, + "p99": 21.01890637646956 + }, + "itl_ms": { + "mean": 17.792685269469718, + "p50": 17.879439161587523, + "p90": 19.834828564609182, + "p95": 20.16044198291448, + "p99": 20.8988025100466 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_512.jsonl" + }, + { + "name": "c32_i4096_o1024", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 113.81290011497913, + "request_throughput": 5.62326414100196, + "input_token_throughput": 23032.88992154403, + "output_token_throughput": 1212.692057407954, + "total_token_throughput": 24245.58197895198, + "total_input_tokens": 2621440, + "total_output_tokens": 138020, + "e2e_ms": { + "mean": 5215.721413190931, + "p50": 4851.276391505962, + "p90": 7216.958626796259, + "p95": 8904.339508028348, + "p99": 13457.617856655856 + }, + "ttft_ms": { + "mean": 339.673375241091, + "p50": 247.79371902695857, + "p90": 392.1883804141544, + "p95": 542.3683993751182, + "p99": 2422.6248358923476 + }, + "tpot_ms": { + "mean": 22.719399348508016, + "p50": 22.914273851038725, + "p90": 25.531362948015392, + "p95": 26.101945052613324, + "p99": 27.20651141262886 + }, + "itl_ms": { + "mean": 22.60461888887343, + "p50": 22.75339210173309, + "p90": 25.422672764186185, + "p95": 25.91771771805469, + "p99": 27.09015545979034 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_1024.jsonl" + }, + { + "name": "c32_i4096_o128", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 85.65506197494688, + "request_throughput": 7.471829279479042, + "input_token_throughput": 30604.612728746157, + "output_token_throughput": 950.8719989464524, + "total_token_throughput": 31555.48472769261, + "total_input_tokens": 2621440, + "total_output_tokens": 81447, + "e2e_ms": { + "mean": 4002.7959108847426, + "p50": 4027.488345018355, + "p90": 4394.019550387748, + "p95": 4613.657232854166, + "p99": 5507.045013857423 + }, + "ttft_ms": { + "mean": 556.1845627042203, + "p50": 511.0022419830784, + "p90": 1002.5600348075387, + "p95": 1180.2316012704969, + "p99": 2449.125620237319 + }, + "tpot_ms": { + "mean": 27.29475949831838, + "p50": 27.766935429013312, + "p90": 30.37622396083811, + "p95": 31.52356485168882, + "p99": 34.81999862995494 + }, + "itl_ms": { + "mean": 27.283351684511892, + "p50": 27.752212318866768, + "p90": 30.37272536720142, + "p95": 31.523207846206727, + "p99": 34.81966031348612 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_128.jsonl" + }, + { + "name": "c32_i4096_o2048", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 114.274240713974, + "request_throughput": 5.600562261462812, + "input_token_throughput": 22939.90302295168, + "output_token_throughput": 1189.489417306433, + "total_token_throughput": 24129.392440258114, + "total_input_tokens": 2621440, + "total_output_tokens": 135928, + "e2e_ms": { + "mean": 5165.783436226775, + "p50": 4726.68347551371, + "p90": 7188.446018210382, + "p95": 9243.12443975359, + "p99": 13018.90308270814 + }, + "ttft_ms": { + "mean": 341.10774283180945, + "p50": 247.46722800773568, + "p90": 392.88402279489674, + "p95": 550.6020500150042, + "p99": 2435.4554109059973 + }, + "tpot_ms": { + "mean": 22.783776407589954, + "p50": 23.01332679174569, + "p90": 25.46372044856561, + "p95": 26.016316538121043, + "p99": 27.730684264550362 + }, + "itl_ms": { + "mean": 22.657550920079434, + "p50": 22.83503046063882, + "p90": 25.269074608539984, + "p95": 25.845271448447257, + "p99": 27.485328640230005 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_2048.jsonl" + }, + { + "name": "c32_i4096_o256", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 106.208948674961, + "request_throughput": 6.025857594717737, + "input_token_throughput": 24681.91270796385, + "output_token_throughput": 1178.6106685143307, + "total_token_throughput": 25860.52337647818, + "total_input_tokens": 2621440, + "total_output_tokens": 125179, + "e2e_ms": { + "mean": 4937.5405230529395, + "p50": 4875.018366990844, + "p90": 6434.248026396381, + "p95": 6734.163024299778, + "p99": 7405.486856222272 + }, + "ttft_ms": { + "mean": 350.7942819734126, + "p50": 248.04657051572576, + "p90": 422.6160959864501, + "p95": 564.3712449935265, + "p99": 2447.9905850067735 + }, + "tpot_ms": { + "mean": 23.550120668178824, + "p50": 23.852025049090017, + "p90": 25.96803448657071, + "p95": 26.71366147238315, + "p99": 28.321813953149746 + }, + "itl_ms": { + "mean": 23.433249650000544, + "p50": 23.708379496861397, + "p90": 25.880524181822437, + "p95": 26.616908152126097, + "p99": 28.186099664674693 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_256.jsonl" + }, + { + "name": "c32_i4096_o4096", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 114.9426989480271, + "request_throughput": 5.567991754651461, + "input_token_throughput": 22806.494227052382, + "output_token_throughput": 1192.220134503394, + "total_token_throughput": 23998.714361555776, + "total_input_tokens": 2621440, + "total_output_tokens": 137037, + "e2e_ms": { + "mean": 5203.629682052178, + "p50": 4792.196563503239, + "p90": 7185.140736214817, + "p95": 8545.926395323471, + "p99": 12625.456661586657 + }, + "ttft_ms": { + "mean": 341.8584196219854, + "p50": 247.48384198755957, + "p90": 401.0035815590527, + "p95": 550.5504803440999, + "p99": 2448.394843073911 + }, + "tpot_ms": { + "mean": 22.82941362215177, + "p50": 23.063525754888257, + "p90": 25.525615854479813, + "p95": 26.158675644158695, + "p99": 27.229142202418178 + }, + "itl_ms": { + "mean": 22.6960209124615, + "p50": 22.959077084300404, + "p90": 25.33381873888988, + "p95": 26.059027535414014, + "p99": 27.1434838141601 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_4096.jsonl" + }, + { + "name": "c32_i4096_o512", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 112.19549296301557, + "request_throughput": 5.7043289627594165, + "input_token_throughput": 23364.93143146257, + "output_token_throughput": 1200.3958130866804, + "total_token_throughput": 24565.32724454925, + "total_input_tokens": 2621440, + "total_output_tokens": 134679, + "e2e_ms": { + "mean": 5150.640929284236, + "p50": 4812.448643002426, + "p90": 7110.52369940444, + "p95": 8529.089289769758, + "p99": 12164.940070008744 + }, + "ttft_ms": { + "mean": 340.2039304295613, + "p50": 247.1121745184064, + "p90": 393.92301979823975, + "p95": 560.246634390205, + "p99": 2385.055927421201 + }, + "tpot_ms": { + "mean": 22.994319271082635, + "p50": 23.234810646082217, + "p90": 25.56160844078038, + "p95": 26.289083146584108, + "p99": 27.726785424863163 + }, + "itl_ms": { + "mean": 22.878491774019594, + "p50": 23.078625202867357, + "p90": 25.48089463753205, + "p95": 26.23845005583209, + "p99": 27.65327865640478 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 78.27041419100715, + "request_throughput": 8.17678054492182, + "input_token_throughput": 4186.511638999972, + "output_token_throughput": 1690.8815593722238, + "total_token_throughput": 5877.393198372196, + "total_input_tokens": 327680, + "total_output_tokens": 132346, + "e2e_ms": { + "mean": 3610.3891835380637, + "p50": 3672.895397496177, + "p90": 4614.467557013268, + "p95": 4786.530874358141, + "p99": 5755.867604461382 + }, + "ttft_ms": { + "mean": 148.9999322566291, + "p50": 122.67996551236138, + "p90": 199.04762358637527, + "p95": 289.9259683268591, + "p99": 473.49165664054453 + }, + "tpot_ms": { + "mean": 16.81089925890317, + "p50": 16.88569357827005, + "p90": 18.596486083093502, + "p95": 19.2741748259838, + "p99": 22.35552161032398 + }, + "itl_ms": { + "mean": 16.743994123153765, + "p50": 16.83361651378548, + "p90": 18.490313803081033, + "p95": 19.17608061977105, + "p99": 22.35535027858728 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_512_256.jsonl" + }, + { + "name": "c32_i65536_o1024", + "config": { + "concurrency": 32, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 266.5209177720244, + "request_throughput": 0.6003281143465826, + "input_token_throughput": 39343.10330181764, + "output_token_throughput": 131.31427091188561, + "total_token_throughput": 39474.41757272952, + "total_input_tokens": 10485760, + "total_output_tokens": 34998, + "e2e_ms": { + "mean": 46303.99489910378, + "p50": 40970.881291490514, + "p90": 78206.72828421228, + "p95": 103438.82800290309, + "p99": 127750.74539481307 + }, + "ttft_ms": { + "mean": 8656.403181048518, + "p50": 5203.471615008311, + "p90": 22916.200463398112, + "p95": 33860.81311377059, + "p99": 43057.064939973636 + }, + "tpot_ms": { + "mean": 171.13581258550425, + "p50": 177.9552146543067, + "p90": 228.0412552804808, + "p95": 244.27115008225184, + "p99": 281.95110766554944 + }, + "itl_ms": { + "mean": 170.32872628814005, + "p50": 176.05291277692726, + "p90": 227.71504553612323, + "p95": 243.7796027544607, + "p99": 281.8399921541697 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_65536_1024.jsonl" + }, + { + "name": "c4_i512_o256", + "config": { + "concurrency": 4, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "metrics": { + "success": 80, + "failed": 0, + "duration_s": 43.39664837101009, + "request_throughput": 1.843460336292739, + "input_token_throughput": 943.8516921818824, + "output_token_throughput": 379.1767479212128, + "total_token_throughput": 1323.0284401030951, + "total_input_tokens": 40960, + "total_output_tokens": 16455, + "e2e_ms": { + "mean": 1720.266553755937, + "p50": 1804.6963794913609, + "p90": 2158.3282727748156, + "p95": 2165.1579623343423, + "p99": 2253.321928069927 + }, + "ttft_ms": { + "mean": 118.070524858922, + "p50": 114.08401752123609, + "p90": 120.62304747523739, + "p95": 125.54392934544008, + "p99": 241.45657837681935 + }, + "tpot_ms": { + "mean": 7.8335008228886505, + "p50": 7.83555123474636, + "p90": 8.170451865484264, + "p95": 8.290225660170474, + "p99": 8.677201510073528 + }, + "itl_ms": { + "mean": 7.794213050172942, + "p50": 7.793371932429836, + "p90": 8.115348610206611, + "p95": 8.234429535969335, + "p99": 8.634192133473178 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_4_512_256.jsonl" + }, + { + "name": "c64_i512_o256", + "config": { + "concurrency": 64, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 1280 + }, + "metrics": { + "success": 1280, + "failed": 0, + "duration_s": 109.33125554397702, + "request_throughput": 11.707539565253938, + "input_token_throughput": 5994.2602574100165, + "output_token_throughput": 2402.359679244252, + "total_token_throughput": 8396.619936654268, + "total_input_tokens": 655360, + "total_output_tokens": 262653, + "e2e_ms": { + "mean": 5151.599965186915, + "p50": 5233.160602510907, + "p90": 6570.279945392394, + "p95": 6720.288137154421, + "p99": 6878.420427516685 + }, + "ttft_ms": { + "mean": 171.44839711986606, + "p50": 132.15252198278904, + "p90": 210.84870708873493, + "p95": 299.06158958619926, + "p99": 771.50551313709 + }, + "tpot_ms": { + "mean": 24.389172122073912, + "p50": 24.77321306780196, + "p90": 26.27769079484252, + "p95": 26.718942784647773, + "p99": 27.576043138567123 + }, + "itl_ms": { + "mean": 24.284871248079536, + "p50": 24.67090953529502, + "p90": 26.175376179470668, + "p95": 26.56671499095787, + "p99": 27.389117637935758 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_64_512_256.jsonl" + }, + { + "name": "c8_i1024_o1024", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 47.79919548804173, + "request_throughput": 3.3473366730623813, + "input_token_throughput": 3427.6727532158784, + "output_token_throughput": 696.0995820175289, + "total_token_throughput": 4123.772335233407, + "total_input_tokens": 163840, + "total_output_tokens": 33273, + "e2e_ms": { + "mean": 2052.0071145107067, + "p50": 2064.3975179700647, + "p90": 2597.689549595816, + "p95": 2788.772759854328, + "p99": 3052.1448046178557 + }, + "ttft_ms": { + "mean": 129.12154367477342, + "p50": 122.31972347944975, + "p90": 124.8896159871947, + "p95": 192.22247132274794, + "p99": 281.036811490776 + }, + "tpot_ms": { + "mean": 9.273661335882394, + "p50": 9.290390007233619, + "p90": 9.797906786052058, + "p95": 9.954364221454668, + "p99": 10.238029201305554 + }, + "itl_ms": { + "mean": 9.22141272050547, + "p50": 9.236384602551324, + "p90": 9.75097784670255, + "p95": 9.913460638994195, + "p99": 10.132330933263706 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_1024.jsonl" + }, + { + "name": "c8_i1024_o128", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 31.03394475398818, + "request_throughput": 5.155644932294286, + "input_token_throughput": 5279.380410669349, + "output_token_throughput": 659.9225513336686, + "total_token_throughput": 5939.3029620030175, + "total_input_tokens": 163840, + "total_output_tokens": 20480, + "e2e_ms": { + "mean": 1315.4882694318076, + "p50": 1366.9841965020169, + "p90": 1380.8117090375163, + "p95": 1387.6815518015064, + "p99": 1458.9439847879112 + }, + "ttft_ms": { + "mean": 187.55042687007517, + "p50": 205.4987929877825, + "p90": 253.20741288596764, + "p95": 269.2095974576661, + "p99": 292.6897038030438 + }, + "tpot_ms": { + "mean": 8.881400335131751, + "p50": 9.162143610260177, + "p90": 9.883002829362132, + "p95": 9.902873972781904, + "p99": 9.973658800444118 + }, + "itl_ms": { + "mean": 8.880977010622292, + "p50": 9.16174857090704, + "p90": 9.88247430792599, + "p95": 9.902345587171277, + "p99": 9.97331234991301 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_128.jsonl" + }, + { + "name": "c8_i1024_o2048", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 47.63014482904691, + "request_throughput": 3.359217163295819, + "input_token_throughput": 3439.838375214919, + "output_token_throughput": 684.7554236305822, + "total_token_throughput": 4124.593798845501, + "total_input_tokens": 163840, + "total_output_tokens": 32615, + "e2e_ms": { + "mean": 2064.275790168176, + "p50": 2025.2659794932697, + "p90": 2682.616906025214, + "p95": 2867.5991577445534, + "p99": 3177.5962958741 + }, + "ttft_ms": { + "mean": 139.63929469391587, + "p50": 123.93425550544634, + "p90": 188.980170188006, + "p95": 191.21294881624635, + "p99": 283.84840004029684 + }, + "tpot_ms": { + "mean": 9.488485002558638, + "p50": 9.326192293000322, + "p90": 10.394212695334996, + "p95": 10.925130294149847, + "p99": 11.765321442654237 + }, + "itl_ms": { + "mean": 9.439770830290232, + "p50": 9.27684741342917, + "p90": 10.347793105855219, + "p95": 10.861698161034582, + "p99": 11.710557326254763 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_2048.jsonl" + }, + { + "name": "c8_i1024_o256", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 47.67866635101382, + "request_throughput": 3.3557985624444346, + "input_token_throughput": 3436.337727943101, + "output_token_throughput": 684.9394603359244, + "total_token_throughput": 4121.277188279026, + "total_input_tokens": 163840, + "total_output_tokens": 32657, + "e2e_ms": { + "mean": 2050.7159092150687, + "p50": 2055.9715820127167, + "p90": 2524.6206711861305, + "p95": 2669.197603108478, + "p99": 2870.3536581806834 + }, + "ttft_ms": { + "mean": 134.21947423594247, + "p50": 122.28806095663458, + "p90": 190.7292490301188, + "p95": 195.34277862985618, + "p99": 290.7564730977174 + }, + "tpot_ms": { + "mean": 9.439803513870618, + "p50": 9.293550548206369, + "p90": 10.193549946169153, + "p95": 10.826949428327007, + "p99": 11.661622554928408 + }, + "itl_ms": { + "mean": 9.387716787724182, + "p50": 9.246371719530723, + "p90": 10.10971836153746, + "p95": 10.739477732631839, + "p99": 11.613613014141018 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_256.jsonl" + }, + { + "name": "c8_i1024_o4096", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.021476240013726, + "request_throughput": 3.3318425947655594, + "input_token_throughput": 3411.806817039933, + "output_token_throughput": 683.4233882350682, + "total_token_throughput": 4095.230205275001, + "total_input_tokens": 163840, + "total_output_tokens": 32819, + "e2e_ms": { + "mean": 2050.123825306946, + "p50": 2050.3479074977804, + "p90": 2538.145926152356, + "p95": 2647.510393260746, + "p99": 2950.3177008335465 + }, + "ttft_ms": { + "mean": 132.58988224442874, + "p50": 122.42957402486354, + "p90": 142.4261361011302, + "p95": 194.91657257894985, + "p99": 285.38757893256843 + }, + "tpot_ms": { + "mean": 9.387625075467776, + "p50": 9.359307527011389, + "p90": 10.081905747530234, + "p95": 10.450971488236782, + "p99": 11.053297382297417 + }, + "itl_ms": { + "mean": 9.337072189720585, + "p50": 9.312614422785066, + "p90": 9.942381647700445, + "p95": 10.406373012197426, + "p99": 11.00866874531187 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_4096.jsonl" + }, + { + "name": "c8_i1024_o512", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.88774840300903, + "request_throughput": 3.272803621083765, + "input_token_throughput": 3351.3509079897754, + "output_token_throughput": 699.8481443234996, + "total_token_throughput": 4051.199052313275, + "total_input_tokens": 163840, + "total_output_tokens": 34214, + "e2e_ms": { + "mean": 2097.309012806363, + "p50": 2095.9312209743075, + "p90": 2660.76910031843, + "p95": 2846.3267137063667, + "p99": 3059.683581781573 + }, + "ttft_ms": { + "mean": 128.42690491161193, + "p50": 122.46163000236265, + "p90": 125.48143291496672, + "p95": 130.8216415811331, + "p99": 286.1712126014754 + }, + "tpot_ms": { + "mean": 9.250483832974629, + "p50": 9.276897661059472, + "p90": 9.808938170245048, + "p95": 9.963068685422419, + "p99": 10.231402362643065 + }, + "itl_ms": { + "mean": 9.202953803540257, + "p50": 9.222722470294482, + "p90": 9.769427792921025, + "p95": 9.900009820334152, + "p99": 10.176064141791748 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_512.jsonl" + }, + { + "name": "c8_i131072_o512", + "config": { + "concurrency": 8, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 593.70801256696, + "request_throughput": 0.26949274157211206, + "input_token_throughput": 35322.95262333987, + "output_token_throughput": 49.96900727637406, + "total_token_throughput": 35372.92163061625, + "total_input_tokens": 20971520, + "total_output_tokens": 29667, + "e2e_ms": { + "mean": 26223.813140128914, + "p50": 25749.483308027266, + "p90": 38876.46534329397, + "p95": 43791.96180334258, + "p99": 52427.09982689234 + }, + "ttft_ms": { + "mean": 8837.25882775725, + "p50": 6957.265121513046, + "p90": 12366.58897944144, + "p95": 12856.652719894191, + "p99": 21274.84452185161 + }, + "tpot_ms": { + "mean": 95.9775316206848, + "p50": 94.03290558605093, + "p90": 149.15055402178177, + "p95": 174.739404388061, + "p99": 250.30791511469616 + }, + "itl_ms": { + "mean": 95.50966075588461, + "p50": 93.91736107077944, + "p90": 148.93209375100713, + "p95": 174.60800165421526, + "p99": 249.69256104941496 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_131072_512.jsonl" + }, + { + "name": "c8_i16384_o1024", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 117.03195754898479, + "request_throughput": 1.3671479427576911, + "input_token_throughput": 22399.35189414201, + "output_token_throughput": 381.4684547279648, + "total_token_throughput": 22780.820348869976, + "total_input_tokens": 2621440, + "total_output_tokens": 44644, + "e2e_ms": { + "mean": 5043.960333102223, + "p50": 4365.808989998186, + "p90": 7655.828198097879, + "p95": 9664.448152296247, + "p99": 11942.893994341719 + }, + "ttft_ms": { + "mean": 880.5494676955277, + "p50": 782.9482170054689, + "p90": 1082.6903782086447, + "p95": 1325.487311300822, + "p99": 2519.2373008432323 + }, + "tpot_ms": { + "mean": 15.139856510889135, + "p50": 15.27841200283051, + "p90": 18.022824161834254, + "p95": 18.881136649500213, + "p99": 22.888170543028938 + }, + "itl_ms": { + "mean": 15.010650016559842, + "p50": 15.1465002130646, + "p90": 17.968179468660793, + "p95": 18.85452186357664, + "p99": 22.829326886115528 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_1024.jsonl" + }, + { + "name": "c8_i16384_o128", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 83.84433197398903, + "request_throughput": 1.9082983456727485, + "input_token_throughput": 31265.56009550231, + "output_token_throughput": 241.22083775769588, + "total_token_throughput": 31506.780933260008, + "total_input_tokens": 2621440, + "total_output_tokens": 20225, + "e2e_ms": { + "mean": 3640.27295542146, + "p50": 3633.8590744999237, + "p90": 3677.6862947270274, + "p95": 3713.6206373630557, + "p99": 5332.837568135582 + }, + "ttft_ms": { + "mean": 1766.892234862462, + "p50": 1687.3688155028503, + "p90": 2631.7813128232956, + "p95": 2637.4724537250586, + "p99": 2676.544717006618 + }, + "tpot_ms": { + "mean": 14.952028106677584, + "p50": 15.45864561090379, + "p90": 20.86819931141837, + "p95": 23.249104839698937, + "p99": 26.077070266717733 + }, + "itl_ms": { + "mean": 14.950050644874448, + "p50": 15.458261126898126, + "p90": 20.867791510470617, + "p95": 23.248591416115467, + "p99": 26.076698547013414 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_128.jsonl" + }, + { + "name": "c8_i16384_o2048", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 111.6486262770486, + "request_throughput": 1.4330673411329817, + "input_token_throughput": 23479.375317122773, + "output_token_throughput": 353.0003128045817, + "total_token_throughput": 23832.375629927355, + "total_input_tokens": 2621440, + "total_output_tokens": 39412, + "e2e_ms": { + "mean": 4778.323991763318, + "p50": 4169.615545513807, + "p90": 7071.359802276127, + "p95": 8910.118855611656, + "p99": 13501.66908308048 + }, + "ttft_ms": { + "mean": 858.3292873940081, + "p50": 782.5817215198185, + "p90": 952.6779550826175, + "p95": 1105.0728265632642, + "p99": 2515.795475198537 + }, + "tpot_ms": { + "mean": 15.798579513726072, + "p50": 16.022473646922144, + "p90": 18.945479346672712, + "p95": 19.890483962991468, + "p99": 21.084441579797186 + }, + "itl_ms": { + "mean": 15.70931575959753, + "p50": 15.979023863701155, + "p90": 18.913301621034854, + "p95": 19.76568012612783, + "p99": 21.016233379075533 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_2048.jsonl" + }, + { + "name": "c8_i16384_o256", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 101.76510345499264, + "request_throughput": 1.572248192827345, + "input_token_throughput": 25759.71439128322, + "output_token_throughput": 329.16981227081504, + "total_token_throughput": 26088.88420355404, + "total_input_tokens": 2621440, + "total_output_tokens": 33498, + "e2e_ms": { + "mean": 4427.855411094424, + "p50": 4454.245780507335, + "p90": 5497.165809822036, + "p95": 5958.67853332893, + "p99": 6667.5567474006675 + }, + "ttft_ms": { + "mean": 867.7743797547009, + "p50": 781.0635255009402, + "p90": 1016.5140870318279, + "p95": 1267.3269381921277, + "p99": 2511.917921541025 + }, + "tpot_ms": { + "mean": 17.06346602830906, + "p50": 17.093144427779265, + "p90": 21.41804815869456, + "p95": 21.934133880069854, + "p99": 23.133364661873244 + }, + "itl_ms": { + "mean": 16.998988221217065, + "p50": 17.06275551950803, + "p90": 21.301843049957412, + "p95": 21.847790275915557, + "p99": 23.01043882594314 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_256.jsonl" + }, + { + "name": "c8_i16384_o512", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 109.64312597998651, + "request_throughput": 1.4592798095633035, + "input_token_throughput": 23908.840399885165, + "output_token_throughput": 355.69033308224544, + "total_token_throughput": 24264.53073296741, + "total_input_tokens": 2621440, + "total_output_tokens": 38999, + "e2e_ms": { + "mean": 4755.6579099466035, + "p50": 4298.210411972832, + "p90": 7776.077105075818, + "p95": 8684.670029362314, + "p99": 10122.991236373782 + }, + "ttft_ms": { + "mean": 908.3466928703274, + "p50": 778.5314369830303, + "p90": 1214.549721876392, + "p95": 1361.7619198863397, + "p99": 2515.331039446754 + }, + "tpot_ms": { + "mean": 15.817801648689121, + "p50": 16.17100685840291, + "p90": 19.34598438045577, + "p95": 20.351946538990223, + "p99": 22.15198565504161 + }, + "itl_ms": { + "mean": 15.714668041430865, + "p50": 15.998185230128716, + "p90": 19.235813678199687, + "p95": 20.317084004124045, + "p99": 22.094216228598047 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_512.jsonl" + }, + { + "name": "c8_i2048_o1024", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 53.1054894879926, + "request_throughput": 3.0128712030076805, + "input_token_throughput": 6170.36022375973, + "output_token_throughput": 677.8395293416718, + "total_token_throughput": 6848.199753101401, + "total_input_tokens": 327680, + "total_output_tokens": 35997, + "e2e_ms": { + "mean": 2286.0924268738017, + "p50": 2064.6599415049423, + "p90": 3458.0778610950797, + "p95": 3870.0411044963403, + "p99": 5178.540953937917 + }, + "ttft_ms": { + "mean": 150.15353084418166, + "p50": 138.43873603036627, + "p90": 145.44060388579965, + "p95": 153.03919319703712, + "p99": 410.55034047400113 + }, + "tpot_ms": { + "mean": 9.488409542087934, + "p50": 9.50819875088937, + "p90": 10.10307734654145, + "p95": 10.240756052118074, + "p99": 10.652978840145037 + }, + "itl_ms": { + "mean": 9.435480404556735, + "p50": 9.475025016652458, + "p90": 10.058684681432286, + "p95": 10.183135112300214, + "p99": 10.599880392631377 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_1024.jsonl" + }, + { + "name": "c8_i2048_o128", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 33.34751936199609, + "request_throughput": 4.79795808087426, + "input_token_throughput": 9826.218149630484, + "output_token_throughput": 612.3993745475884, + "total_token_throughput": 10438.617524178073, + "total_input_tokens": 327680, + "total_output_tokens": 20422, + "e2e_ms": { + "mean": 1411.2315205631603, + "p50": 1428.1102574896067, + "p90": 1452.4172595120035, + "p95": 1530.949007422896, + "p99": 1595.8804498135578 + }, + "ttft_ms": { + "mean": 228.93475683886209, + "p50": 150.7378289825283, + "p90": 379.75782579160295, + "p95": 388.7717862293357, + "p99": 404.45577931881417 + }, + "tpot_ms": { + "mean": 9.333638432390616, + "p50": 9.567953153480373, + "p90": 10.250220458961905, + "p95": 10.326194063064598, + "p99": 11.017481187701328 + }, + "itl_ms": { + "mean": 9.331251454181302, + "p50": 9.567510058890706, + "p90": 10.249756787184948, + "p95": 10.32570392511213, + "p99": 11.017141694314851 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_128.jsonl" + }, + { + "name": "c8_i2048_o2048", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 51.12094455503393, + "request_throughput": 3.1298326232558753, + "input_token_throughput": 6409.897212428033, + "output_token_throughput": 672.6988380071643, + "total_token_throughput": 7082.596050435197, + "total_input_tokens": 327680, + "total_output_tokens": 34389, + "e2e_ms": { + "mean": 2202.4928271144745, + "p50": 2062.3054489842616, + "p90": 3023.14127808786, + "p95": 3406.000318186131, + "p99": 4277.616084612091 + }, + "ttft_ms": { + "mean": 150.70150687461137, + "p50": 138.49432798451744, + "p90": 144.4377199979499, + "p95": 165.44266314012995, + "p99": 417.99726528290194 + }, + "tpot_ms": { + "mean": 9.560429413090715, + "p50": 9.588894799229887, + "p90": 10.169284756728882, + "p95": 10.25023479426485, + "p99": 10.675569194381492 + }, + "itl_ms": { + "mean": 9.507038628210164, + "p50": 9.556331128852783, + "p90": 10.125696983636297, + "p95": 10.207410243990491, + "p99": 10.55116458914923 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_2048.jsonl" + }, + { + "name": "c8_i2048_o256", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.032057252014056, + "request_throughput": 3.3311086210718357, + "input_token_throughput": 6822.1104559551195, + "output_token_throughput": 658.8516463902457, + "total_token_throughput": 7480.962102345365, + "total_input_tokens": 327680, + "total_output_tokens": 31646, + "e2e_ms": { + "mean": 2051.772922800228, + "p50": 2031.0940880153794, + "p90": 2680.799270607531, + "p95": 2715.9889496513642, + "p99": 2745.1213573251152 + }, + "ttft_ms": { + "mean": 148.5712587586022, + "p50": 137.2895505046472, + "p90": 142.76428538141772, + "p95": 147.17790617141873, + "p99": 403.30891362857074 + }, + "tpot_ms": { + "mean": 9.655633524071195, + "p50": 9.704072993671705, + "p90": 10.211875773304342, + "p95": 10.47933470059539, + "p99": 10.984298463796051 + }, + "itl_ms": { + "mean": 9.60982244299022, + "p50": 9.667875145867418, + "p90": 10.210687953745627, + "p95": 10.38416295380234, + "p99": 10.939679769297733 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_256.jsonl" + }, + { + "name": "c8_i2048_o4096", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 53.121544424968306, + "request_throughput": 3.0119606222291315, + "input_token_throughput": 6168.495354325261, + "output_token_throughput": 661.0500575637386, + "total_token_throughput": 6829.545411889, + "total_input_tokens": 327680, + "total_output_tokens": 35116, + "e2e_ms": { + "mean": 2250.238640499447, + "p50": 2060.5107424780726, + "p90": 3155.8005181024782, + "p95": 3788.7343745154812, + "p99": 4649.511583077255 + }, + "ttft_ms": { + "mean": 150.99034105223836, + "p50": 137.72946450626478, + "p90": 144.35978347901255, + "p95": 219.6447414258729, + "p99": 421.1795450851787 + }, + "tpot_ms": { + "mean": 9.576291877122781, + "p50": 9.656859571750678, + "p90": 10.295353155562944, + "p95": 10.442840379462343, + "p99": 10.576213952237916 + }, + "itl_ms": { + "mean": 9.521656809451594, + "p50": 9.60966083004923, + "p90": 10.244706533607456, + "p95": 10.41720936334391, + "p99": 10.524654688477527 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_4096.jsonl" + }, + { + "name": "c8_i2048_o512", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 50.97147612500703, + "request_throughput": 3.139010524388221, + "input_token_throughput": 6428.693553947077, + "output_token_throughput": 669.0408556417945, + "total_token_throughput": 7097.734409588871, + "total_input_tokens": 327680, + "total_output_tokens": 34102, + "e2e_ms": { + "mean": 2190.124648256824, + "p50": 2051.997427508468, + "p90": 3092.7031756960787, + "p95": 3392.3705204302682, + "p99": 4269.132630236562 + }, + "ttft_ms": { + "mean": 148.41127000145207, + "p50": 137.25388501188718, + "p90": 143.0399636097718, + "p95": 147.71226262673738, + "p99": 403.41484271222726 + }, + "tpot_ms": { + "mean": 9.604414580719462, + "p50": 9.644402433341643, + "p90": 10.206372463942852, + "p95": 10.483934730782448, + "p99": 11.082257841628142 + }, + "itl_ms": { + "mean": 9.547872710361627, + "p50": 9.566959381900393, + "p90": 10.16867574821862, + "p95": 10.445367728095432, + "p99": 10.99869270011619 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_512.jsonl" + }, + { + "name": "c8_i262144_o128", + "config": { + "concurrency": 8, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 1333.5599152920186, + "request_throughput": 0.11997961108853795, + "input_token_throughput": 31451.935169193694, + "output_token_throughput": 12.611356870543945, + "total_token_throughput": 31464.546526064238, + "total_input_tokens": 41943040, + "total_output_tokens": 16818, + "e2e_ms": { + "mean": 59143.646888742296, + "p50": 59151.90828900086, + "p90": 59696.19761038339, + "p95": 61777.268645277945, + "p99": 74358.74609683407 + }, + "ttft_ms": { + "mean": 36128.44722178088, + "p50": 31553.45409095753, + "p90": 57830.65334893763, + "p95": 57997.25315597025, + "p99": 58592.27338458295 + }, + "tpot_ms": { + "mean": 215.94794021037245, + "p50": 256.24419141826854, + "p90": 391.74129292216816, + "p95": 392.1382184028426, + "p99": 446.9436230443744 + }, + "itl_ms": { + "mean": 215.80289381987473, + "p50": 255.43096417095512, + "p90": 391.74069926375523, + "p95": 392.1376104767094, + "p99": 446.94327607237824 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_262144_128.jsonl" + }, + { + "name": "c8_i4096_o1024", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 60.1764503219747, + "request_throughput": 2.6588474252621817, + "input_token_throughput": 10890.639053873896, + "output_token_throughput": 578.3824039765639, + "total_token_throughput": 11469.02145785046, + "total_input_tokens": 655360, + "total_output_tokens": 34805, + "e2e_ms": { + "mean": 2550.258225586367, + "p50": 2352.0323729899246, + "p90": 3329.045555065386, + "p95": 4642.49045046745, + "p99": 7535.8905110671185 + }, + "ttft_ms": { + "mean": 253.10391337516194, + "p50": 232.87756598438136, + "p90": 238.5712873074226, + "p95": 364.2884655972013, + "p99": 731.9332470308291 + }, + "tpot_ms": { + "mean": 10.57177228837578, + "p50": 10.64941300418702, + "p90": 11.587295458922837, + "p95": 11.887076822407384, + "p99": 12.167304394979784 + }, + "itl_ms": { + "mean": 10.507668801966785, + "p50": 10.586497942778886, + "p90": 11.535050429966855, + "p95": 11.721179619833572, + "p99": 11.979766313497109 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_1024.jsonl" + }, + { + "name": "c8_i4096_o128", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 40.354094801004976, + "request_throughput": 3.9649012272235473, + "input_token_throughput": 16240.23542670765, + "output_token_throughput": 506.0205191244052, + "total_token_throughput": 16746.255945832054, + "total_input_tokens": 655360, + "total_output_tokens": 20420, + "e2e_ms": { + "mean": 1721.3472404528147, + "p50": 1727.0378270186484, + "p90": 1749.9196672462858, + "p95": 1910.0958328694105, + "p99": 2181.081341035315 + }, + "ttft_ms": { + "mean": 376.50707827706356, + "p50": 259.70822700764984, + "p90": 686.7596228956245, + "p95": 695.3964474290842, + "p99": 727.2619360673707 + }, + "tpot_ms": { + "mean": 10.61910417605264, + "p50": 10.8540513521434, + "p90": 11.925979451263528, + "p95": 11.98689676731858, + "p99": 13.733626962350641 + }, + "itl_ms": { + "mean": 10.616558701551055, + "p50": 10.81030888415592, + "p90": 11.925486865403922, + "p95": 11.986494040658412, + "p99": 13.733118322708156 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_128.jsonl" + }, + { + "name": "c8_i4096_o2048", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.13547281298088, + "request_throughput": 2.800361878928021, + "input_token_throughput": 11470.282256089175, + "output_token_throughput": 593.2216595274147, + "total_token_throughput": 12063.50391561659, + "total_input_tokens": 655360, + "total_output_tokens": 33894, + "e2e_ms": { + "mean": 2478.8010334064893, + "p50": 2387.5871959899087, + "p90": 3337.4612528132275, + "p95": 3843.5530389077044, + "p99": 5407.4504248407875 + }, + "ttft_ms": { + "mean": 254.35324738755298, + "p50": 234.77283149259165, + "p90": 237.6235678908415, + "p95": 361.27230101847096, + "p99": 732.140513684717 + }, + "tpot_ms": { + "mean": 10.518332535585891, + "p50": 10.590422963040899, + "p90": 11.502042432841588, + "p95": 11.682779447372331, + "p99": 12.363350102408123 + }, + "itl_ms": { + "mean": 10.459223410056985, + "p50": 10.526562515166122, + "p90": 11.435158975709735, + "p95": 11.629216193575695, + "p99": 12.304858903853528 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_2048.jsonl" + }, + { + "name": "c8_i4096_o256", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 54.73408304201439, + "request_throughput": 2.923224271011949, + "input_token_throughput": 11973.526614064944, + "output_token_throughput": 575.2357260800701, + "total_token_throughput": 12548.762340145013, + "total_input_tokens": 655360, + "total_output_tokens": 31485, + "e2e_ms": { + "mean": 2342.5611934311746, + "p50": 2375.0031819799915, + "p90": 2998.0281604104675, + "p95": 3122.9925069375895, + "p99": 3247.983481837436 + }, + "ttft_ms": { + "mean": 252.75953688651498, + "p50": 232.09355198196135, + "p90": 238.3180244883988, + "p95": 366.29044984583743, + "p99": 732.2054903308162 + }, + "tpot_ms": { + "mean": 10.626070005954862, + "p50": 10.651171180348502, + "p90": 11.678796063279654, + "p95": 12.020385139763485, + "p99": 12.587150601270418 + }, + "itl_ms": { + "mean": 10.578348915412478, + "p50": 10.631525566033428, + "p90": 11.625460455067769, + "p95": 11.963682290983403, + "p99": 12.537233502371484 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_256.jsonl" + }, + { + "name": "c8_i4096_o4096", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 61.97299993701745, + "request_throughput": 2.5817694828813584, + "input_token_throughput": 10574.927801882044, + "output_token_throughput": 602.6011333637771, + "total_token_throughput": 11177.528935245822, + "total_input_tokens": 655360, + "total_output_tokens": 37345, + "e2e_ms": { + "mean": 2715.5006801924173, + "p50": 2296.5276565228123, + "p90": 3336.566875816788, + "p95": 3797.2634706238773, + "p99": 5487.115504498359 + }, + "ttft_ms": { + "mean": 251.6203917311941, + "p50": 233.67508800583892, + "p90": 237.69924038788304, + "p95": 276.3132474210587, + "p99": 733.8285815808922 + }, + "tpot_ms": { + "mean": 10.484129967546775, + "p50": 10.615274145465921, + "p90": 11.663851770502971, + "p95": 11.806859850762217, + "p99": 12.225997983203529 + }, + "itl_ms": { + "mean": 10.422449648022756, + "p50": 10.50003365653224, + "p90": 11.607559697366497, + "p95": 11.752852474078695, + "p99": 12.169660355465783 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_4096.jsonl" + }, + { + "name": "c8_i4096_o512", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.591448950988706, + "request_throughput": 2.7781902159843677, + "input_token_throughput": 11379.46712467197, + "output_token_throughput": 586.7537736158985, + "total_token_throughput": 11966.22089828787, + "total_input_tokens": 655360, + "total_output_tokens": 33792, + "e2e_ms": { + "mean": 2482.8999182365806, + "p50": 2328.508917009458, + "p90": 3322.9401516378853, + "p95": 4048.3197444671596, + "p99": 5636.745985752786 + }, + "ttft_ms": { + "mean": 255.020999507542, + "p50": 233.46155596664175, + "p90": 243.00072313053533, + "p95": 376.2320582260145, + "p99": 733.9472911530174 + }, + "tpot_ms": { + "mean": 10.53575250735599, + "p50": 10.566613968387138, + "p90": 11.549565030957893, + "p95": 11.703975537785546, + "p99": 12.172998480346749 + }, + "itl_ms": { + "mean": 10.477811694518088, + "p50": 10.5075818050335, + "p90": 11.473276370174169, + "p95": 11.617491250124376, + "p99": 11.994851243780436 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_512.jsonl" + }, + { + "name": "c8_i512_o256", + "config": { + "concurrency": 8, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.34384520503227, + "request_throughput": 3.309625027165714, + "input_token_throughput": 1694.5280139088457, + "output_token_throughput": 686.9954150139231, + "total_token_throughput": 2381.5234289227687, + "total_input_tokens": 81920, + "total_output_tokens": 33212, + "e2e_ms": { + "mean": 2075.1491860271926, + "p50": 2173.637369007338, + "p90": 2598.430469626328, + "p95": 2669.6843849582365, + "p99": 2871.4014900499023 + }, + "ttft_ms": { + "mean": 125.67628707365657, + "p50": 116.35541348368861, + "p90": 131.15600130986414, + "p95": 194.69460808322768, + "p99": 258.2202711654827 + }, + "tpot_ms": { + "mean": 9.433366300506062, + "p50": 9.411735510561925, + "p90": 10.00254407070816, + "p95": 10.430004206536266, + "p99": 10.950972265487408 + }, + "itl_ms": { + "mean": 9.394523221295401, + "p50": 9.36648559752587, + "p90": 9.957444830914309, + "p95": 10.423651138739814, + "p99": 10.936330572939495 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_512_256.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 298.66033907805104, + "request_throughput": 0.5357256356632812, + "input_token_throughput": 35109.3152588288, + "output_token_throughput": 107.56366278551818, + "total_token_throughput": 35216.87892161432, + "total_input_tokens": 10485760, + "total_output_tokens": 32125, + "e2e_ms": { + "mean": 13001.392204627336, + "p50": 12542.045173991937, + "p90": 19688.615291757742, + "p95": 21336.68931651919, + "p99": 30533.505789504845 + }, + "ttft_ms": { + "mean": 3847.522275053052, + "p50": 3268.898812500993, + "p90": 5648.875438020332, + "p95": 5884.526047899271, + "p99": 9844.517542416572 + }, + "tpot_ms": { + "mean": 46.162656343687104, + "p50": 44.22957419128011, + "p90": 71.19218689531144, + "p95": 82.22329133258519, + "p99": 101.30540995954283 + }, + "itl_ms": { + "mean": 45.949579851959996, + "p50": 44.21014147421841, + "p90": 71.18646031193735, + "p95": 81.51694651542725, + "p99": 101.30496182853231 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 274.1067653300124, + "request_throughput": 0.5837141589970137, + "input_token_throughput": 38254.29112402829, + "output_token_throughput": 70.29742581070911, + "total_token_throughput": 38324.588549839, + "total_input_tokens": 10485760, + "total_output_tokens": 19269, + "e2e_ms": { + "mean": 12089.844919632378, + "p50": 12085.347602027468, + "p90": 12178.208685596474, + "p95": 12347.214980889112, + "p99": 12881.315556197076 + }, + "ttft_ms": { + "mean": 7223.655334466821, + "p50": 7265.156039007707, + "p90": 11034.40593361156, + "p95": 11101.746441455907, + "p99": 11262.27838820545 + }, + "tpot_ms": { + "mean": 40.500587106675695, + "p50": 40.765876922386276, + "p90": 72.42243729535613, + "p95": 72.53750583942507, + "p99": 72.63441232998551 + }, + "itl_ms": { + "mean": 40.4976505026951, + "p50": 40.765389234094, + "p90": 72.42207835310077, + "p95": 72.53694979015502, + "p99": 72.63400630344259 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_128.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 291.00532128399936, + "request_throughput": 0.549818124610347, + "input_token_throughput": 36032.8806144637, + "output_token_throughput": 98.87104425805563, + "total_token_throughput": 36131.751658721754, + "total_input_tokens": 10485760, + "total_output_tokens": 28772, + "e2e_ms": { + "mean": 12843.240155881358, + "p50": 12733.343144471291, + "p90": 17990.28853759519, + "p95": 18584.33374123124, + "p99": 21293.288793755928 + }, + "ttft_ms": { + "mean": 3804.886426846133, + "p50": 3264.9704460054636, + "p90": 5579.19040689012, + "p95": 5718.145171192011, + "p99": 9852.35885454632 + }, + "tpot_ms": { + "mean": 50.924454910755436, + "p50": 49.41619370337081, + "p90": 72.27905076555149, + "p95": 78.36170262279879, + "p99": 96.49280662046667 + }, + "itl_ms": { + "mean": 50.71989161569722, + "p50": 49.07127746615946, + "p90": 72.20096208478664, + "p95": 76.45671442829546, + "p99": 95.44811846482862 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_256.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 297.0369087840081, + "request_throughput": 0.5386535991604491, + "input_token_throughput": 35301.20227457919, + "output_token_throughput": 111.90191864058855, + "total_token_throughput": 35413.104193219784, + "total_input_tokens": 10485760, + "total_output_tokens": 33239, + "e2e_ms": { + "mean": 13139.025081034924, + "p50": 12636.850299983053, + "p90": 20860.50389088923, + "p95": 22983.13999388484, + "p99": 31590.46198525988 + }, + "ttft_ms": { + "mean": 3902.294257010726, + "p50": 3372.0412499969825, + "p90": 5779.39000299084, + "p95": 6055.86706744216, + "p99": 9835.604973481379 + }, + "tpot_ms": { + "mean": 44.23024008462845, + "p50": 44.65291751845124, + "p90": 66.10532247314792, + "p95": 73.93369533170166, + "p99": 83.81848196608775 + }, + "itl_ms": { + "mean": 44.01114226693558, + "p50": 44.573540121602136, + "p90": 65.53879755658635, + "p95": 73.88402448965356, + "p99": 83.77087468636375 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_512.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md deleted file mode 100644 index 288b634..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# dsv4_h200_vllm_tp4_custom_bench_no_fp8 - -## 目的 - -当 vLLM 使用 **TP=4**（Tensor Parallel = 4）部署时，单张 H200 上可以同时运行 **2 个独立服务**（8 卡 / 4 = 2 服务），每个服务独占 4 张 GPU： - -> **注意**：本实验变体 **不启用 `--kv-cache-dtype fp8`**，用于对比 FP8 KV Cache 与默认精度的性能差异。 - -| 服务 | 端口 | GPU 组 | -|------|------|--------| -| 1 | 30005 | 0, 1, 2, 3 | -| 2 | 30006 | 4, 5, 6, 7 | - -相比 TP=2 方案，TP=4 的每个服务拥有更大的 tensor-parallel 宽度，适合更高并发的单服务场景。本实验同样使用 **自定义压测客户端 `bench_client.py`**，通过负载均衡将请求均匀分发到 2 个服务上。 - -## 与 TP=2 方案的区别 - -| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | -|------|---------------|---------------| -| 服务数量 | 4 | 2 | -| 每服务 GPU | 2 | 4 | -| 单服务吞吐 | 较低 | 较高 | -| 集群总吞吐 | 相近 | 相近 | -| 适用场景 | 低并发、低延迟 | 高并发、大 batch | - -## 文件说明 - -| 文件 | 说明 | -|------|------| -| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡（复用 TP=2 版本） | -| `config.env` | 实验配置（2 服务、端口 30005/30006、场景等） | -| `run_bench.sh` | 编排脚本：启动 2 个服务 → 运行压测 → 解析结果 | -| `start_server.sh` | 同时启动 2 个 vLLM TP=4 服务（每服务 4 GPU） | -| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | - -## 快速运行 - -```bash -# 完整流程（自动启动 2 个服务、压测、生成报告） -bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh - -# 跳过服务管理（已有 2 个服务在运行） -SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh - -# 单独运行压测客户端（多服务，带健康检查） -python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006 \ - --model deepseek-v4-flash \ - --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ - --lb-strategy round_robin \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file /tmp/test.jsonl - -# 单独运行压测客户端（单服务，向后兼容） -python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ - --output-file /tmp/test_single.jsonl - -# 解析已有结果 -python3 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py \ - experiments/dsv4_h200_vllm_tp4_custom_bench/results/ -``` - -## 场景设计 - -本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： - -| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | -|------|----------|----------|--------|------| -| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | -| 2 | 512 | 256 | 40 | 低并发，2 服务各 1 req | -| 4 | 512 | 256 | 80 | 中低并发 | -| 8 | 512 | 256 | 160 | 中等并发 | -| 16 | 512 | 256 | 320 | 中高并发 | -| 32 | 512 | 256 | 640 | 高并发 | -| 64 | 512 | 256 | 1280 | 很高并发 | -| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | -| 1 | 2048 | 512 | 20 | 长输入基线 | -| 8 | 2048 | 512 | 160 | 长输入中并发 | -| 32 | 2048 | 512 | 640 | 长输入高并发 | -| 128 | 2048 | 512 | 2560 | 长输入极限并发 | -| 1 | 4096 | 1024 | 20 | 超长输入基线 | -| 8 | 4096 | 1024 | 160 | 超长输入中并发 | -| 32 | 4096 | 1024 | 640 | 超长输入高并发 | -| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | - -## 健康检查与故障自动跳过 - -同 TP=2 版本，详见原 README。 - -## 负载均衡策略 - -同 TP=2 版本，支持 `round_robin`（默认）、`random`、`least_conn`。 - -## 输出格式 - -同 TP=2 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 - -## 单独启动/停止 2 个服务 - -```bash -# 启动 2 个服务 -bash experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh - -# 停止（run_bench.sh 会自动停止，但也可以手动） -for port in 30005 30006; do - pid_file="experiments/dsv4_h200_vllm_tp4_custom_bench/server_port${port}.pid" - [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true - rm -f "$pid_file" -done -pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true -``` - -## 注意事项 - -- 2 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 -- 每个服务占用约 4 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 -- TP=4 相比 TP=2 单服务延迟更低（通信更少），但服务数量减半，低并发下可能无法充分利用集群。 -- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 -- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py deleted file mode 100755 index d8edc36..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py +++ /dev/null @@ -1,617 +0,0 @@ -#!/usr/bin/env python3 -"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. - -Sends concurrent requests to multiple inference servers and records per-request -latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. - -Designed for TP=2 on 8-GPU setups where 4 independent services run on -(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via -round-robin or random load balancing. Unhealthy services are automatically -skipped with periodic retry. - -Usage: - # Single service (backward compatible) - python3 bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --output-file results.jsonl - - # Multi-service (4 services on ports 30005-30008) - python3 bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --lb-strategy round_robin \ - --output-file results.jsonl - -The output JSONL contains one object per request with raw timing data. -A final summary line ("completed" field) aggregates all requests. -""" - -import argparse -import asyncio -import json -import random -import sys -import time -import urllib.request -from dataclasses import asdict, dataclass, field -from typing import Any - -# Try aiohttp first; fall back to urllib for stdlib-only environments. -try: - import aiohttp - HAS_AIOHTTP = True -except ImportError: - HAS_AIOHTTP = False - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - -@dataclass -class RequestResult: - request_id: int - input_tokens: int = 0 - output_tokens: int = 0 - first_token_time_ms: float = 0.0 # TTFT - e2e_latency_ms: float = 0.0 # total wall time - inter_token_latencies: list[float] = field(default_factory=list) - success: bool = False - error: str = "" - target_port: int = 0 # which service handled this request - retry_count: int = 0 # how many times this request was retried - - @property - def tpot_ms(self) -> float: - """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" - if self.output_tokens <= 1: - return 0.0 - return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) - - @property - def mean_itl_ms(self) -> float: - if not self.inter_token_latencies: - return 0.0 - return sum(self.inter_token_latencies) / len(self.inter_token_latencies) - - def to_dict(self) -> dict[str, Any]: - d = asdict(self) - d["tpot_ms"] = self.tpot_ms - d["mean_itl_ms"] = self.mean_itl_ms - return d - - -# --------------------------------------------------------------------------- -# Health checker -# --------------------------------------------------------------------------- - -class HealthChecker: - """Periodically check service health and maintain a healthy-port list.""" - - def __init__( - self, - host: str, - ports: list[int], - check_interval: float = 5.0, - max_failures: int = 2, - recovery_interval: float = 10.0, - ): - self.host = host - self.all_ports = ports - self.check_interval = check_interval - self.max_failures = max_failures - self.recovery_interval = recovery_interval - - # port -> consecutive failure count - self._failures: dict[int, int] = {p: 0 for p in ports} - # port -> last healthy timestamp - self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} - self._lock = asyncio.Lock() - self._task: asyncio.Task | None = None - - async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: - url = f"http://{self.host}:{port}/health" - try: - if HAS_AIOHTTP and session is not None: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: - return resp.status == 200 - else: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - return resp.status == 200 - except Exception: - return False - - async def _check_all(self, session: aiohttp.ClientSession | None) -> None: - checks = [self._check_one(p, session) for p in self.all_ports] - results = await asyncio.gather(*checks, return_exceptions=True) - now = time.monotonic() - async with self._lock: - for port, ok in zip(self.all_ports, results): - if isinstance(ok, Exception): - ok = False - if ok: - self._failures[port] = 0 - self._last_healthy[port] = now - else: - self._failures[port] += 1 - - async def start(self, session: aiohttp.ClientSession | None) -> None: - async def _loop() -> None: - while True: - await asyncio.sleep(self.check_interval) - await self._check_all(session) - self._task = asyncio.create_task(_loop()) - - async def stop(self) -> None: - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - @property - async def healthy_ports(self) -> list[int]: - now = time.monotonic() - async with self._lock: - healthy = [] - for p in self.all_ports: - if self._failures[p] < self.max_failures: - healthy.append(p) - elif now - self._last_healthy[p] > self.recovery_interval: - # Give it another chance after recovery_interval. - self._failures[p] = max(0, self._failures[p] - 1) - healthy.append(p) - return healthy - - async def is_healthy(self, port: int) -> bool: - ports = await self.healthy_ports - return port in ports - - -# --------------------------------------------------------------------------- -# Load balancer with failover -# --------------------------------------------------------------------------- - -class LoadBalancer: - """Distribute requests across healthy backend ports with auto-failover.""" - - def __init__( - self, - ports: list[int], - health_checker: HealthChecker, - strategy: str = "round_robin", - ): - self.all_ports = ports - self.health_checker = health_checker - self.strategy = strategy - self._idx = 0 - self._lock = asyncio.Lock() - - def _next_round_robin(self, healthy: list[int]) -> int: - if not healthy: - return self.all_ports[self._idx % len(self.all_ports)] - for _ in range(len(self.all_ports)): - port = self.all_ports[self._idx % len(self.all_ports)] - self._idx += 1 - if port in healthy: - return port - # Fallback: all unhealthy, pick first healthy anyway. - return healthy[0] if healthy else self.all_ports[0] - - def _next_random(self, healthy: list[int]) -> int: - if healthy: - return random.choice(healthy) - return random.choice(self.all_ports) - - async def next_port(self) -> int: - healthy = await self.health_checker.healthy_ports - async with self._lock: - if self.strategy == "round_robin": - return self._next_round_robin(healthy) - elif self.strategy in ("random", "least_conn"): - return self._next_random(healthy) - else: - return self._next_round_robin(healthy) - - @property - def port_count(self) -> int: - return len(self.all_ports) - - -# --------------------------------------------------------------------------- -# Token helpers -# --------------------------------------------------------------------------- - -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: - tokens.extend(words) - return " ".join(tokens[:needed]) - - -# --------------------------------------------------------------------------- -# aiohttp-based async request (preferred) -# --------------------------------------------------------------------------- - -async def send_request_aiohttp( - session: aiohttp.ClientSession, - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - } - headers = {"Content-Type": "application/json"} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - first_token_received = False - last_token_time = 0.0 - - try: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status != 200: - text = await resp.text() - result.error = f"HTTP {resp.status}: {text[:200]}" - return result - - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or not line.startswith("data: "): - continue - data_str = line[len("data: "):] - if data_str == "[DONE]": - break - try: - chunk = json.loads(data_str) - except json.JSONDecodeError: - continue - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - now = time.perf_counter() - if not first_token_received: - result.first_token_time_ms = (now - start_time) * 1000 - first_token_received = True - else: - result.inter_token_latencies.append((now - last_token_time) * 1000) - last_token_time = now - result.output_tokens += 1 - - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.input_tokens = len(prompt.split()) # approximate - result.success = True - - except asyncio.TimeoutError: - result.error = "timeout" - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# urllib-based synchronous request (fallback, no aiohttp) -# --------------------------------------------------------------------------- - -def send_request_urllib( - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": False, - } - data = json.dumps(payload).encode("utf-8") - headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - - try: - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=600) as resp: - body = json.loads(resp.read().decode("utf-8")) - - choices = body.get("choices", []) - if not choices: - result.error = "no choices in response" - return result - - content = choices[0].get("message", {}).get("content", "") - result.output_tokens = len(content.split()) if content else 0 - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.first_token_time_ms = result.e2e_latency_ms - result.input_tokens = len(prompt.split()) - result.success = True - - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# Unified send_request wrapper with retry & failover -# --------------------------------------------------------------------------- - -async def send_request_with_retry( - session: aiohttp.ClientSession | None, - host: str, - lb: LoadBalancer, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, - max_retries: int = 2, -) -> RequestResult: - """Send a request, retrying on failure with a different port each time.""" - result = RequestResult(request_id=request_id) - for attempt in range(max_retries + 1): - port = await lb.next_port() - if HAS_AIOHTTP and session is not None: - result = await send_request_aiohttp( - session, host, port, model, request_id, prompt, max_tokens, temperature - ) - else: - loop = asyncio.get_event_loop() - result = await loop.run_in_executor( - None, send_request_urllib, - host, port, model, request_id, prompt, max_tokens, temperature - ) - result.retry_count = attempt - if result.success: - return result - # If failed, try another port on next iteration (unless last attempt). - if attempt < max_retries: - await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish - return result - - -# --------------------------------------------------------------------------- -# Benchmark orchestration -# --------------------------------------------------------------------------- - -async def run_benchmark( - host: str, - ports: list[int], - lb: LoadBalancer, - health_checker: HealthChecker, - model: str, - concurrency: int, - input_len: int, - output_len: int, - num_prompts: int, - temperature: float, - output_file: str, -) -> dict[str, Any]: - """Run the benchmark and write a JSONL file.""" - prompts = [make_prompt(input_len) for _ in range(num_prompts)] - results: list[RequestResult] = [] - semaphore = asyncio.Semaphore(concurrency) - - start = time.perf_counter() - - if HAS_AIOHTTP: - connector = aiohttp.TCPConnector(limit=concurrency * 2) - timeout = aiohttp.ClientTimeout(total=600, sock_read=300) - async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: - await health_checker.start(session) - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - session, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - else: - await health_checker.start(None) - from concurrent.futures import ThreadPoolExecutor - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - None, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - - duration = time.perf_counter() - start - - # Write JSONL - with open(output_file, "w", encoding="utf-8") as f: - for r in results: - f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") - - # Compute summary - successful = [r for r in results if r.success] - failed = len(results) - len(successful) - retried = [r for r in results if r.retry_count > 0] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - ttfts = [r.first_token_time_ms for r in successful] - tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] - e2es = [r.e2e_latency_ms for r in successful] - itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] - - total_input_tokens = sum(r.input_tokens for r in successful) - total_output_tokens = sum(r.output_tokens for r in successful) - - # Per-port breakdown - per_port_stats: dict[int, dict[str, Any]] = {} - for port in ports: - port_results = [r for r in successful if r.target_port == port] - port_failed = [r for r in results if r.target_port == port and not r.success] - if port_results or port_failed: - port_ttfts = [r.first_token_time_ms for r in port_results] - port_e2es = [r.e2e_latency_ms for r in port_results] - per_port_stats[port] = { - "count": len(port_results), - "failed": len(port_failed), - "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, - "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, - "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, - } - - # Health-check stats - healthy_at_end = await health_checker.healthy_ports - - summary = { - "completed": len(successful), - "failed": failed, - "retried": len(retried), - "duration": duration, - "request_throughput": len(successful) / duration if duration > 0 else 0.0, - "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, - "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, - "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, - "total_input_tokens": total_input_tokens, - "total_output_tokens": total_output_tokens, - "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, - "median_ttft_ms": percentile(ttfts, 50), - "p90_ttft_ms": percentile(ttfts, 90), - "p95_ttft_ms": percentile(ttfts, 95), - "p99_ttft_ms": percentile(ttfts, 99), - "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, - "median_tpot_ms": percentile(tpots, 50), - "p90_tpot_ms": percentile(tpots, 90), - "p95_tpot_ms": percentile(tpots, 95), - "p99_tpot_ms": percentile(tpots, 99), - "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, - "median_e2e_latency_ms": percentile(e2es, 50), - "p90_e2e_latency_ms": percentile(e2es, 90), - "p95_e2e_latency_ms": percentile(e2es, 95), - "p99_e2e_latency_ms": percentile(e2es, 99), - "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, - "median_itl_ms": percentile(itls, 50), - "p90_itl_ms": percentile(itls, 90), - "p95_itl_ms": percentile(itls, 95), - "p99_itl_ms": percentile(itls, 99), - "per_port_stats": per_port_stats, - "healthy_ports_at_end": healthy_at_end, - "lb_strategy": lb.strategy, - "num_services": lb.port_count, - } - - # Append summary as the last line - with open(output_file, "a", encoding="utf-8") as f: - f.write(json.dumps(summary, ensure_ascii=False) + "\n") - - return summary - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30005, - help="Single service port (backward compatible)") - parser.add_argument("--ports", default="", - help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") - parser.add_argument("--model", default="deepseek-v4-flash") - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--input-len", type=int, default=512) - parser.add_argument("--output-len", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=10) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--lb-strategy", default="round_robin", - choices=["round_robin", "random", "least_conn"], - help="Load balancing strategy across services") - parser.add_argument("--health-check-interval", type=float, default=5.0, - help="Seconds between health checks (default: 5)") - parser.add_argument("--health-max-failures", type=int, default=2, - help="Consecutive failures before marking a port unhealthy (default: 2)") - parser.add_argument("--health-recovery-interval", type=float, default=10.0, - help="Seconds before retrying an unhealthy port (default: 10)") - parser.add_argument("--request-max-retries", type=int, default=2, - help="Max retries per request on failure (default: 2)") - parser.add_argument("--output-file", required=True) - args = parser.parse_args() - - # Determine ports: --ports takes precedence, else fall back to --port. - if args.ports: - ports = [int(p.strip()) for p in args.ports.split(",")] - else: - ports = [args.port] - - health_checker = HealthChecker( - host=args.host, - ports=ports, - check_interval=args.health_check_interval, - max_failures=args.health_max_failures, - recovery_interval=args.health_recovery_interval, - ) - lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) - - summary = asyncio.run(run_benchmark( - host=args.host, - ports=ports, - lb=lb, - health_checker=health_checker, - model=args.model, - concurrency=args.concurrency, - input_len=args.input_len, - output_len=args.output_len, - num_prompts=args.num_prompts, - temperature=args.temperature, - output_file=args.output_file, - )) - - print(json.dumps(summary, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env deleted file mode 100644 index 8f784be..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env +++ /dev/null @@ -1,84 +0,0 @@ -# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench -# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. -# -# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: -# Service 1: port 30005, GPUs 0,1 -# Service 2: port 30006, GPUs 2,3 -# Service 3: port 30007, GPUs 4,5 -# Service 4: port 30008, GPUs 6,7 -# -# bench_client.py distributes requests across all 4 services via round-robin -# load balancing, giving accurate cluster-wide throughput even at low -# per-service concurrency. - -EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench_no_fp8" -MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" - -# Primary port (backward compatible); multi-service ports are in PORTS. -PORT="30005" -PORTS="30005,30006" - -BACKEND="vllm" -ENGINE="vllm" - -# Native virtual environments on the host. -VENV_SERVER="/data/user1/yy/envs/vllm" -VENV_CLIENT="/data/user1/yy/envs/vllm" - -# Load balancing strategy: round_robin | random | least_conn -LB_STRATEGY="round_robin" - -# Benchmark scenarios: "concurrency input_len output_len num_prompts". -# For multi-service we include low-concurrency scenarios to show how -# requests spread across 2 services. -SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" -) - -# Server start script bundled with this experiment. -SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py deleted file mode 100755 index eea1a2e..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. - -Reads JSONL files produced by bench_client.py (one request per line, -last line is the summary) and generates: - - results.json (appended scenarios, following the standard schema) - - report.md (human-readable summary) - -Usage: - python3 parse_results.py -""" - -import json -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: - """Read the summary JSON (last line) and all per-request lines.""" - requests: list[dict] = [] - summary: dict | None = None - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - # The summary line has "completed" and "duration" keys. - if "completed" in obj and "duration" in obj: - summary = obj - else: - requests.append(obj) - return summary, requests - - -def compute_metrics(summary: dict, requests: list[dict]) -> dict: - completed = summary.get("completed", 0) - total = len(requests) - failed = total - completed if total > 0 else 0 - duration_s = summary.get("duration", 0.0) - - # Extract per-request metrics for percentiles. - ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] - tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] - e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] - itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": summary.get("request_throughput", 0.0), - "input_token_throughput": summary.get("input_throughput", 0.0), - "output_token_throughput": summary.get("output_throughput", 0.0), - "total_token_throughput": summary.get("total_throughput", 0.0), - "total_input_tokens": summary.get("total_input_tokens", 0), - "total_output_tokens": summary.get("total_output_tokens", 0), - "e2e_ms": { - "mean": summary.get("mean_e2e_latency_ms", 0.0), - "p50": percentile(e2es, 50), - "p90": percentile(e2es, 90), - "p95": percentile(e2es, 95), - "p99": percentile(e2es, 99), - }, - "ttft_ms": { - "mean": summary.get("mean_ttft_ms", 0.0), - "p50": percentile(ttfts, 50), - "p90": percentile(ttfts, 90), - "p95": percentile(ttfts, 95), - "p99": percentile(ttfts, 99), - }, - "tpot_ms": { - "mean": summary.get("mean_tpot_ms", 0.0), - "p50": percentile(tpots, 50), - "p90": percentile(tpots, 90), - "p95": percentile(tpots, 95), - "p99": percentile(tpots, 99), - }, - "itl_ms": { - "mean": summary.get("mean_itl_ms", 0.0), - "p50": percentile(itls, 50), - "p90": percentile(itls, 90), - "p95": percentile(itls, 95), - "p99": percentile(itls, 99), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: - ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms - tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") - f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") - f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") - f.write(f"- **Result root**: `{result_root}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - - summary, requests = parse_jsonl(jsonl_path) - if summary is None: - continue - - metrics = compute_metrics(summary, requests) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": "random", - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - for s in scenarios: - append_scenario(results_json, s) - - generate_report(result_root, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh deleted file mode 100755 index 0ec7941..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env bash -# Custom benchmark orchestrator for multi-service vLLM TP=4 on 8x H200. -# -# This runs 2 independent vLLM services (TP=4 each) on GPU groups (0,1,2,3) and (4,5,6,7). -# bench_client.py distributes requests across all 2 services via round-robin LB, -# giving accurate cluster-wide throughput even at low per-service concurrency. -# -# Usage: -# bash run_bench.sh -# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" - -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -ensure_result_root "$RESULT_ROOT" -log_init "${LOG_DIR}/orchestrator.log" - -log "experiment=${EXPERIMENT_NAME}" -log "run_id=${RUN_ID}" -log "result_root=${RESULT_ROOT}" -log "platform=${PLATFORM}" -log "chip=${CHIP}" -log "accelerator=${ACCELERATOR}" -log "engine=${ENGINE}" -log "hardware=${HARDWARE}" -log "model=${MODEL_PATH}" -log "services=${PORTS}" -log "lb_strategy=${LB_STRATEGY}" -log "server_start_script=${SERVER_START_SCRIPT}" - -# Write initial metadata for this run. -METADATA_JSON="${RESULT_ROOT}/results.json" -write_metadata_json \ - "$METADATA_JSON" \ - "$EXPERIMENT_NAME" \ - "$RUN_ID" \ - "$MODEL_PATH" \ - "$BACKEND" \ - "$ENGINE" \ - "$HARDWARE" \ - "$ACCELERATOR" \ - "$CHIP" \ - "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_SERVER" \ - "H200 2x vLLM TP=4 multi-service benchmark using bench_client.py (no FP8 KV cache)" - -# Update metadata with config. -"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' -import json, sys -path = sys.argv[1] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 4, - "num_services": 2, - "ports": "30005,30006", - "gpu_groups": "0,1,2,3|4,5,6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", "2 512 256 40", "4 512 256 80", - "8 512 256 160", "16 512 256 320", "32 512 256 640", - "64 512 256 1280", "128 512 256 2560", - "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", - "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" - ] -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY - -# --------------------------------------------------------------------------- -# Server helpers (multi-service) -# --------------------------------------------------------------------------- - -is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -are_all_services_healthy() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - for p in "${PORT_LIST[@]}"; do - if ! is_server_healthy "$p"; then - return 1 - fi - done - return 0 -} - -stop_all_servers() { - log "stopping all vllm services" - # Kill by PID files first. - for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do - [[ -f "$pid_file" ]] || continue - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "killing pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - done - # Fallback: kill any remaining vllm processes for this model. - pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true - sleep 2 -} - -start_all_servers() { - log "starting all vllm services with ${SERVER_START_SCRIPT}" - if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" - exit 1 - fi - bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - - if ! are_all_services_healthy "$PORTS"; then - log "error: not all vllm services became healthy" - exit 1 - fi - log "all vllm services are healthy" -} - -# --------------------------------------------------------------------------- -# Pre-benchmark health check with detailed reporting -# --------------------------------------------------------------------------- - -check_services_detailed() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy_count=0 - local unhealthy_ports="" - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - ((healthy_count++)) - else - unhealthy_ports="${unhealthy_ports}${p} " - fi - done - echo "$healthy_count" - if [[ -n "$unhealthy_ports" ]]; then - echo "unhealthy: ${unhealthy_ports}" >&2 - fi -} - -wait_for_all_services() { - local ports="$1" - local max_wait="${2:-240}" - local interval="${3:-5}" - local elapsed=0 - while (( elapsed < max_wait )); do - local healthy_count - healthy_count="$(check_services_detailed "$ports" | head -1)" - IFS=',' read -ra PORT_LIST <<< "$ports" - if (( healthy_count == ${#PORT_LIST[@]} )); then - return 0 - fi - log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" - sleep "$interval" - ((elapsed += interval)) - done - return 1 -} - -# --------------------------------------------------------------------------- -# Mid-benchmark health check (called between scenarios) -# --------------------------------------------------------------------------- - -check_and_report_services() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy=() - local unhealthy=() - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - healthy+=("$p") - else - unhealthy+=("$p") - fi - done - log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" - if [[ ${#unhealthy[@]} -gt 0 ]]; then - log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" - fi - return ${#unhealthy[@]} -} - -on_exit() { - local code=$? - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_all_servers - fi - exit "$code" -} -trap on_exit EXIT - -if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming services are already running" - if ! wait_for_all_services "$PORTS"; then - log "error: not all healthy services found on ports ${PORTS}" - exit 1 - fi -else - stop_all_servers - start_all_servers -fi - -log "===== BENCHMARK START =====" - -for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len num_prompts <<< "$scenario" - # Fallback for legacy 3-field scenarios. - if [[ -z "${num_prompts:-}" ]]; then - num_prompts="$NUM_PROMPTS" - fi - - output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" - - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ - --host 127.0.0.1 \ - --ports "$PORTS" \ - --model "$SERVED_MODEL_NAME" \ - --concurrency "$concurrency" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num-prompts "$num_prompts" \ - --lb-strategy "$LB_STRATEGY" \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file "$output_file" \ - > "$detail_log" 2>&1 || { - log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" - # After failure, check service health before continuing. - check_and_report_services "$PORTS" - continue - } - - log "finished scenario: output=${output_file}" - - # Between scenarios, report service health so we can spot degradation early. - check_and_report_services "$PORTS" || true -done - -log "===== BENCHMARK DONE =====" - -log "parsing results" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { - log "WARNING: parser failed; see ${LOG_DIR}/parse.log" -} - -log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh deleted file mode 100755 index 25cce7d..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# Start 2 independent vLLM TP=4 services on 8x H200. -# Each service runs on 4 GPUs: (0,1,2,3) and (4,5,6,7). -# This maximizes per-service throughput for higher-concurrency workloads. -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -VENV="${VENV_SERVER}" -export PATH="$VENV/bin:$PATH" -VLLM="$VENV/bin/vllm" - -TP=4 -LOG_DIR="${SCRIPT_DIR}/logs" -mkdir -p "$LOG_DIR" - -# 2 services, each on 4 GPUs. -# Port and GPU mapping. -SERVICES=( - "30005:0,1,2,3" - "30006:4,5,6,7" -) - -# Kill any existing vLLM processes for this model. -pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true -sleep 2 - -# Remove old PID files. -rm -f "${SCRIPT_DIR}"/*.pid - -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - GPUS="${svc##*:}" - LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - - echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" - echo "Log: ${LOG}" - - CUDA_VISIBLE_DEVICES="${GPUS}" \ - nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --block-size 256 \ - --served-model-name "$SERVED_MODEL_NAME" \ - --port "$PORT" \ - > "$LOG" 2>&1 & - PID=$! - echo $PID > "$PID_FILE" - echo "PID: $PID" -done - -echo "" -echo "Waiting for all 2 services to become healthy..." - -MAX_WAIT=240 -all_healthy=0 -for i in $(seq 1 $MAX_WAIT); do - all_healthy=1 - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then - all_healthy=0 - break - fi - done - - if [[ "$all_healthy" -eq 1 ]]; then - echo "All 2 services are healthy!" - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - echo " - http://127.0.0.1:${PORT}" - done - exit 0 - fi - - # Check if any process died early. - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - if [[ -f "$PID_FILE" ]]; then - PID="$(cat "$PID_FILE")" - if ! kill -0 "$PID" 2>/dev/null; then - echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - tail -200 $LOG 2>/dev/null || true - exit 1 - fi - fi - done - - if (( i % 10 == 0 )); then - echo "Waiting... (${i}/${MAX_WAIT})" - fi - sleep 5 -done - -echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - echo "--- Last 50 lines of port ${PORT} ---" - tail -50 $LOG 2>/dev/null || true -done -exit 1 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py index d8edc36..9deaa5b 100755 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py @@ -228,16 +228,22 @@ class LoadBalancer: # Token helpers # --------------------------------------------------------------------------- -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: +import uuid +def make_prompt(token_len: int) -> str: + prefix = str(uuid.uuid4()) + + words = [ + "the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "benchmark", "performance", "latency" + ] + + tokens = [prefix] + + while len(tokens) < token_len: tokens.extend(words) - return " ".join(tokens[:needed]) + + return " ".join(tokens[:token_len]) # --------------------------------------------------------------------------- @@ -440,7 +446,36 @@ async def run_benchmark( return await send_request_with_retry( session, host, lb, model, idx, prompt, output_len, temperature ) + print("Starting warmup...") + warmup_per_service = 10 + warmup_requests = warmup_per_service * len(ports) + warmup_prompts = [ + make_prompt(input_len) + for _ in range(warmup_requests) + ] + + warmup_tasks = [ + send_request_with_retry( + session=session, + host=host, + lb=lb, + model=model, + request_id=-1, + prompt=p, + max_tokens=output_len, + temperature=temperature, + ) + for p in warmup_prompts + ] + + await asyncio.gather(*warmup_tasks) + + print("Warmup finished.") + + # 给 CUDA / Scheduler 一点时间稳定 + await asyncio.sleep(2) + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] results = await asyncio.gather(*tasks) await health_checker.stop() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env index 0cc0836..d559ca6 100644 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env @@ -32,51 +32,148 @@ LB_STRATEGY="round_robin" # Benchmark scenarios: "concurrency input_len output_len num_prompts". # For single-service TP=8 we cover standard concurrency gradients. +# concurrency input_len output_len num_prompts SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + + "1 1024 128 20" + "1 1024 256 20" + "1 1024 512 20" + "1 1024 1024 20" + "1 1024 2048 20" + "1 1024 4096 20" + + "8 1024 128 160" + "8 1024 256 160" + "8 1024 512 160" + "8 1024 1024 160" + "8 1024 2048 160" + "8 1024 4096 160" + + "32 1024 128 640" + "32 1024 256 640" + "32 1024 512 640" + "32 1024 1024 640" + "32 1024 2048 640" + "32 1024 4096 640" + + "128 1024 128 2560" + "128 1024 256 2560" + "128 1024 512 2560" + "128 1024 1024 2560" + "128 1024 2048 2560" + "128 1024 4096 2560" + + "1 2048 128 20" + "1 2048 256 20" + "1 2048 512 20" + "1 2048 1024 20" + "1 2048 2048 20" + "1 2048 4096 20" + + "8 2048 128 160" + "8 2048 256 160" + "8 2048 512 160" + "8 2048 1024 160" + "8 2048 2048 160" + "8 2048 4096 160" + + "32 2048 128 640" + "32 2048 256 640" + "32 2048 512 640" + "32 2048 1024 640" + "32 2048 2048 640" + "32 2048 4096 640" + + "128 2048 128 2560" + "128 2048 256 2560" + "128 2048 512 2560" + "128 2048 1024 2560" + "128 2048 2048 2560" + "128 2048 4096 2560" + + "1 4096 128 20" + "1 4096 256 20" + "1 4096 512 20" + "1 4096 1024 20" + "1 4096 2048 20" + "1 4096 4096 20" + + "8 4096 128 160" + "8 4096 256 160" + "8 4096 512 160" + "8 4096 1024 160" + "8 4096 2048 160" + "8 4096 4096 160" + + "32 4096 128 640" + "32 4096 256 640" + "32 4096 512 640" + "32 4096 1024 640" + "32 4096 2048 640" + "32 4096 4096 640" + + "128 4096 128 2560" + "128 4096 256 2560" + "128 4096 512 2560" + "128 4096 1024 2560" + "128 4096 2048 2560" + "128 4096 4096 2560" + + "1 16384 128 20" + "1 16384 256 20" + "1 16384 512 20" + "1 16384 1024 20" + "1 16384 2048 20" + + "8 16384 128 160" + "8 16384 256 160" + "8 16384 512 160" + "8 16384 1024 160" + "8 16384 2048 160" + + "32 16384 128 640" + "32 16384 256 640" + "32 16384 512 640" + "32 16384 1024 640" + "32 16384 2048 640" + + "128 16384 128 2560" + "128 16384 256 2560" + "128 16384 512 2560" + "128 16384 1024 2560" + "128 16384 2048 2560" + + + "1 65536 128 20" + "1 65536 256 20" + "1 65536 512 20" + "1 65536 1024 20" + + "8 65536 128 160" + "8 65536 256 160" + "8 65536 512 160" + "8 65536 1024 160" + "32 65536 1024 160" + + "1 131072 128 20" + "1 131072 256 20" + "1 131072 512 20" + + "8 131072 512 160" + + "1 262144 256 20" + + "8 262144 128 160" + + "1 524288 128 20" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md new file mode 100644 index 0000000..c8cb28f --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md @@ -0,0 +1,129 @@ +# H200 vLLM TP=2 Custom Benchmark Report + +- **Client**: `bench_client.py` (async OpenAI API, per-request timing) +- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding) +- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i1024_o1024 | 128 | 1024 | 1024 | 206.75 | 2560 | 0 | 12.38 | 12679.34 | 2588.49 | 15267.83 | 355.51 | 541.00 | 3399.25 | 46.35 | 50.58 | 52.54 | 9995.61 | 13713.98 | 15284.95 | ✅ | +| c128_i1024_o128 | 128 | 1024 | 128 | 133.01 | 2560 | 0 | 19.25 | 19708.67 | 2459.70 | 22168.36 | 1183.32 | 1479.29 | 3426.82 | 41.60 | 47.50 | 49.22 | 6458.71 | 7012.22 | 8759.52 | ✅ | +| c128_i1024_o2048 | 128 | 1024 | 2048 | 208.80 | 2560 | 0 | 12.26 | 12554.51 | 2547.16 | 15101.67 | 351.46 | 493.93 | 3429.21 | 47.17 | 52.36 | 59.29 | 10106.58 | 13980.86 | 15725.90 | ✅ | +| c128_i1024_o256 | 128 | 1024 | 256 | 205.11 | 2560 | 0 | 12.48 | 12780.63 | 2561.23 | 15341.86 | 351.22 | 456.94 | 3426.54 | 46.93 | 50.36 | 51.39 | 9935.31 | 12606.55 | 13028.92 | ✅ | +| c128_i1024_o4096 | 128 | 1024 | 4096 | 208.63 | 2560 | 0 | 12.27 | 12564.96 | 2562.18 | 15127.14 | 352.13 | 475.05 | 3401.06 | 46.82 | 50.35 | 51.59 | 10079.33 | 13938.76 | 15626.71 | ✅ | +| c128_i1024_o512 | 128 | 1024 | 512 | 210.21 | 2560 | 0 | 12.18 | 12470.38 | 2534.38 | 15004.76 | 355.66 | 519.46 | 3397.33 | 47.25 | 52.84 | 55.05 | 10138.54 | 13931.68 | 15398.92 | ✅ | +| c128_i16384_o128 | 128 | 16384 | 128 | 1359.07 | 2560 | 0 | 1.88 | 30861.66 | 240.91 | 31102.57 | 34308.65 | 34726.75 | 53850.14 | 257.93 | 261.13 | 261.25 | 67037.51 | 67882.99 | 86482.77 | ❌ | +| c128_i16384_o256 | 128 | 16384 | 256 | 1383.21 | 2560 | 0 | 1.85 | 30323.03 | 393.12 | 30716.14 | 13674.31 | 16619.00 | 53959.97 | 259.70 | 265.74 | 266.01 | 68522.28 | 82348.90 | 113143.30 | ❌ | +| c128_i16384_o512 | 128 | 16384 | 512 | 1394.58 | 2560 | 0 | 1.84 | 30075.73 | 449.14 | 30524.87 | 6929.19 | 14006.30 | 53929.18 | 256.42 | 266.39 | 266.68 | 68997.86 | 134484.18 | 142208.84 | ❌ | +| c128_i2048_o1024 | 128 | 2048 | 1024 | 274.18 | 2560 | 0 | 9.34 | 19121.84 | 2068.13 | 21189.97 | 541.55 | 764.67 | 6604.67 | 57.69 | 62.18 | 65.53 | 13253.98 | 21801.42 | 28048.56 | ⚠️ | +| c128_i2048_o128 | 128 | 2048 | 128 | 212.99 | 2560 | 0 | 12.02 | 24615.40 | 1530.02 | 26145.42 | 1293.09 | 1997.49 | 6703.31 | 72.33 | 80.38 | 81.78 | 10431.32 | 11370.95 | 15884.14 | ⚠️ | +| c128_i2048_o2048 | 128 | 2048 | 2048 | 272.06 | 2560 | 0 | 9.41 | 19270.69 | 2055.16 | 21325.85 | 542.66 | 696.46 | 6595.59 | 58.02 | 62.67 | 65.23 | 13143.51 | 21512.40 | 28583.36 | ⚠️ | +| c128_i2048_o256 | 128 | 2048 | 256 | 263.83 | 2560 | 0 | 9.70 | 19872.28 | 1986.56 | 21858.84 | 552.74 | 766.94 | 6667.42 | 60.36 | 64.80 | 67.61 | 12841.30 | 16455.00 | 19770.97 | ⚠️ | +| c128_i2048_o4096 | 128 | 2048 | 4096 | 274.18 | 2560 | 0 | 9.34 | 19121.82 | 2075.59 | 21197.41 | 538.85 | 772.70 | 6666.24 | 57.59 | 61.73 | 65.98 | 13271.96 | 22132.76 | 28679.85 | ⚠️ | +| c128_i2048_o512 | 128 | 2048 | 512 | 271.80 | 2560 | 0 | 9.42 | 19289.73 | 2074.26 | 21363.99 | 544.24 | 780.74 | 6725.67 | 57.75 | 61.62 | 65.40 | 13185.12 | 21822.20 | 28335.44 | ⚠️ | +| c128_i4096_o1024 | 128 | 4096 | 1024 | 424.83 | 2560 | 0 | 6.03 | 24682.53 | 1273.81 | 25956.34 | 1014.26 | 1488.91 | 13325.90 | 93.93 | 103.74 | 115.24 | 20697.67 | 35507.95 | 47170.14 | ⚠️ | +| c128_i4096_o128 | 128 | 4096 | 128 | 371.42 | 2560 | 0 | 6.89 | 28231.71 | 873.71 | 29105.42 | 1687.74 | 2056.57 | 13344.42 | 132.08 | 142.11 | 145.58 | 18310.58 | 19462.37 | 30169.28 | ⚠️ | +| c128_i4096_o2048 | 128 | 4096 | 2048 | 425.25 | 2560 | 0 | 6.02 | 24657.77 | 1288.03 | 25945.80 | 1019.46 | 1464.89 | 13332.86 | 93.13 | 101.52 | 111.69 | 20779.35 | 36358.76 | 50447.50 | ⚠️ | +| c128_i4096_o256 | 128 | 4096 | 256 | 413.80 | 2560 | 0 | 6.19 | 25340.14 | 1217.79 | 26557.93 | 1065.75 | 1554.31 | 13333.89 | 98.37 | 108.53 | 114.97 | 20311.65 | 26976.60 | 33054.57 | ⚠️ | +| c128_i4096_o4096 | 128 | 4096 | 4096 | 423.76 | 2560 | 0 | 6.04 | 24744.50 | 1282.56 | 26027.06 | 1040.84 | 1941.31 | 13328.94 | 93.34 | 101.81 | 111.13 | 20723.12 | 35255.54 | 48374.31 | ⚠️ | +| c128_i4096_o512 | 128 | 4096 | 512 | 423.70 | 2560 | 0 | 6.04 | 24748.17 | 1274.42 | 26022.58 | 1032.52 | 1562.56 | 13323.34 | 93.86 | 103.68 | 112.51 | 20669.59 | 35396.55 | 49034.77 | ⚠️ | +| c128_i512_o256 | 128 | 512 | 256 | 198.05 | 2560 | 0 | 12.93 | 6618.16 | 2673.83 | 9291.99 | 290.66 | 501.00 | 1944.87 | 45.16 | 52.68 | 57.19 | 9585.92 | 13135.34 | 14237.26 | ✅ | +| c16_i512_o256 | 16 | 512 | 256 | 70.00 | 320 | 0 | 4.57 | 2340.54 | 954.25 | 3294.79 | 155.99 | 299.17 | 421.01 | 14.77 | 16.43 | 17.44 | 3227.91 | 4110.18 | 4253.24 | ✅ | +| c1_i1024_o1024 | 1 | 1024 | 1024 | 41.00 | 20 | 0 | 0.49 | 499.49 | 102.02 | 601.51 | 113.27 | 116.29 | 137.79 | 6.73 | 6.74 | 6.74 | 1513.65 | 1949.27 | 1998.66 | ✅ | +| c1_i1024_o128 | 1 | 1024 | 128 | 26.62 | 20 | 0 | 0.75 | 769.29 | 96.16 | 865.45 | 112.10 | 114.09 | 127.55 | 6.69 | 6.70 | 6.70 | 961.87 | 964.11 | 977.09 | ✅ | +| c1_i1024_o2048 | 1 | 1024 | 2048 | 40.37 | 20 | 0 | 0.50 | 507.26 | 100.39 | 607.65 | 113.54 | 118.17 | 129.38 | 6.73 | 6.75 | 6.75 | 1470.10 | 1949.87 | 2093.40 | ✅ | +| c1_i1024_o256 | 1 | 1024 | 256 | 39.66 | 20 | 0 | 0.50 | 516.37 | 101.41 | 617.77 | 112.03 | 113.19 | 128.94 | 6.73 | 6.74 | 6.74 | 1457.72 | 1819.60 | 1819.96 | ✅ | +| c1_i1024_o4096 | 1 | 1024 | 4096 | 39.07 | 20 | 0 | 0.51 | 524.19 | 100.38 | 624.57 | 126.46 | 178.17 | 178.35 | 6.73 | 6.74 | 6.75 | 1439.54 | 1773.78 | 1831.83 | ✅ | +| c1_i1024_o512 | 1 | 1024 | 512 | 38.86 | 20 | 0 | 0.51 | 527.06 | 98.62 | 625.68 | 113.09 | 114.42 | 136.00 | 6.73 | 6.74 | 6.75 | 1396.05 | 1797.45 | 1820.39 | ✅ | +| c1_i16384_o1024 | 1 | 16384 | 1024 | 60.02 | 20 | 0 | 0.33 | 5459.87 | 74.28 | 5534.15 | 621.87 | 633.10 | 637.21 | 6.82 | 6.83 | 6.85 | 2136.32 | 2962.66 | 3211.92 | ✅ | +| c1_i16384_o128 | 1 | 16384 | 128 | 41.67 | 20 | 0 | 0.48 | 7862.85 | 60.88 | 7923.72 | 628.49 | 631.58 | 636.36 | 6.77 | 6.81 | 6.81 | 1480.09 | 1482.52 | 1482.98 | ✅ | +| c1_i16384_o2048 | 1 | 16384 | 2048 | 63.80 | 20 | 0 | 0.31 | 5136.27 | 80.76 | 5217.03 | 630.15 | 636.36 | 636.98 | 6.82 | 6.83 | 6.85 | 2381.45 | 4577.47 | 5209.66 | ✅ | +| c1_i16384_o256 | 1 | 16384 | 256 | 56.44 | 20 | 0 | 0.35 | 5805.62 | 73.65 | 5879.27 | 630.48 | 638.61 | 639.04 | 6.82 | 6.85 | 6.85 | 2040.24 | 2358.86 | 2362.47 | ✅ | +| c1_i16384_o512 | 1 | 16384 | 512 | 70.93 | 20 | 0 | 0.28 | 4619.51 | 85.87 | 4705.38 | 621.45 | 629.96 | 645.42 | 6.82 | 6.85 | 6.85 | 2692.18 | 4094.39 | 4094.98 | ✅ | +| c1_i2048_o1024 | 1 | 2048 | 1024 | 43.34 | 20 | 0 | 0.46 | 945.05 | 102.00 | 1047.06 | 123.15 | 127.95 | 148.69 | 6.80 | 6.81 | 6.81 | 1619.33 | 2410.40 | 2903.39 | ✅ | +| c1_i2048_o128 | 1 | 2048 | 128 | 27.02 | 20 | 0 | 0.74 | 1515.96 | 94.12 | 1610.08 | 122.85 | 126.65 | 147.47 | 6.77 | 6.77 | 6.82 | 976.42 | 983.68 | 1006.22 | ✅ | +| c1_i2048_o2048 | 1 | 2048 | 2048 | 43.35 | 20 | 0 | 0.46 | 944.86 | 101.45 | 1046.31 | 123.06 | 128.19 | 147.54 | 6.80 | 6.83 | 6.84 | 1611.88 | 2108.88 | 2496.01 | ✅ | +| c1_i2048_o256 | 1 | 2048 | 256 | 37.22 | 20 | 0 | 0.54 | 1100.59 | 96.68 | 1197.27 | 122.80 | 125.69 | 145.02 | 6.81 | 6.82 | 6.83 | 1340.69 | 1621.05 | 1775.55 | ✅ | +| c1_i2048_o4096 | 1 | 2048 | 4096 | 40.11 | 20 | 0 | 0.50 | 1021.08 | 101.61 | 1122.69 | 123.10 | 127.51 | 146.72 | 6.80 | 6.82 | 6.82 | 1502.66 | 2076.61 | 2137.65 | ✅ | +| c1_i2048_o512 | 1 | 2048 | 512 | 37.02 | 20 | 0 | 0.54 | 1106.44 | 96.52 | 1202.96 | 122.80 | 127.33 | 146.10 | 6.81 | 6.82 | 6.82 | 1331.89 | 1558.94 | 1578.90 | ✅ | +| c1_i4096_o1024 | 1 | 4096 | 1024 | 43.12 | 20 | 0 | 0.46 | 1899.75 | 99.83 | 1999.59 | 186.35 | 190.63 | 190.95 | 6.80 | 6.81 | 6.81 | 1644.18 | 2109.89 | 2311.93 | ✅ | +| c1_i4096_o128 | 1 | 4096 | 128 | 28.91 | 20 | 0 | 0.69 | 2833.86 | 88.56 | 2922.42 | 185.86 | 190.38 | 191.25 | 6.75 | 6.76 | 6.76 | 1043.43 | 1045.36 | 1047.58 | ✅ | +| c1_i4096_o2048 | 1 | 4096 | 2048 | 42.94 | 20 | 0 | 0.47 | 1907.88 | 96.51 | 2004.39 | 186.77 | 193.30 | 193.74 | 6.80 | 6.81 | 6.81 | 1589.25 | 2097.82 | 2452.99 | ✅ | +| c1_i4096_o256 | 1 | 4096 | 256 | 40.43 | 20 | 0 | 0.49 | 2026.29 | 93.65 | 2119.94 | 189.05 | 207.59 | 207.83 | 6.80 | 6.81 | 6.82 | 1469.51 | 1913.76 | 1913.87 | ✅ | +| c1_i4096_o4096 | 1 | 4096 | 4096 | 43.12 | 20 | 0 | 0.46 | 1899.82 | 96.29 | 1996.11 | 187.93 | 205.29 | 207.29 | 6.80 | 6.82 | 6.82 | 1593.63 | 2383.61 | 2729.94 | ✅ | +| c1_i4096_o512 | 1 | 4096 | 512 | 41.37 | 20 | 0 | 0.48 | 1980.16 | 89.19 | 2069.35 | 186.00 | 187.35 | 190.66 | 6.81 | 6.81 | 6.82 | 1434.80 | 2102.19 | 2396.89 | ✅ | +| c1_i512_o256 | 1 | 512 | 256 | 39.88 | 20 | 0 | 0.50 | 256.78 | 96.80 | 353.58 | 118.18 | 176.60 | 184.48 | 6.72 | 6.75 | 6.75 | 1408.44 | 1822.85 | 1871.30 | ✅ | +| c2_i512_o256 | 2 | 512 | 256 | 41.58 | 40 | 0 | 0.96 | 492.58 | 196.05 | 688.62 | 173.55 | 211.08 | 357.86 | 7.61 | 8.16 | 8.40 | 1706.70 | 2108.24 | 2165.31 | ✅ | +| c32_i1024_o1024 | 32 | 1024 | 1024 | 101.71 | 640 | 0 | 6.29 | 6443.13 | 1317.59 | 7760.73 | 197.93 | 311.27 | 984.94 | 22.03 | 23.78 | 24.21 | 4796.09 | 6445.23 | 7146.16 | ✅ | +| c32_i1024_o128 | 32 | 1024 | 128 | 53.75 | 640 | 0 | 11.91 | 12192.47 | 1523.87 | 13716.35 | 681.10 | 1046.63 | 1091.53 | 14.34 | 16.98 | 18.60 | 2502.18 | 2579.44 | 2641.32 | ✅ | +| c32_i1024_o2048 | 32 | 1024 | 2048 | 101.76 | 640 | 0 | 6.29 | 6440.44 | 1316.74 | 7757.18 | 204.19 | 335.85 | 1145.74 | 21.89 | 24.02 | 24.53 | 4767.74 | 6630.19 | 7109.51 | ✅ | +| c32_i1024_o256 | 32 | 1024 | 256 | 99.60 | 640 | 0 | 6.43 | 6579.98 | 1309.14 | 7889.12 | 200.93 | 331.50 | 921.88 | 22.15 | 24.50 | 25.32 | 4694.16 | 6056.40 | 6274.46 | ✅ | +| c32_i1024_o4096 | 32 | 1024 | 4096 | 103.10 | 640 | 0 | 6.21 | 6356.26 | 1309.63 | 7665.88 | 205.97 | 316.61 | 1157.52 | 22.04 | 24.05 | 24.79 | 4833.69 | 6458.13 | 7314.04 | ✅ | +| c32_i1024_o512 | 32 | 1024 | 512 | 103.06 | 640 | 0 | 6.21 | 6359.18 | 1304.32 | 7663.50 | 196.61 | 340.27 | 945.16 | 22.38 | 25.38 | 27.97 | 4872.21 | 6788.10 | 7689.83 | ✅ | +| c32_i16384_o1024 | 32 | 16384 | 1024 | 398.20 | 640 | 0 | 1.61 | 26333.15 | 402.85 | 26735.99 | 1772.26 | 3387.53 | 13518.46 | 70.46 | 85.49 | 101.84 | 19184.30 | 36508.49 | 54198.78 | ❌ | +| c32_i16384_o128 | 32 | 16384 | 128 | 359.20 | 640 | 0 | 1.78 | 29191.93 | 227.55 | 29419.48 | 3438.60 | 3983.92 | 13516.26 | 111.13 | 127.95 | 130.66 | 17518.50 | 19731.45 | 27923.40 | ❌ | +| c32_i16384_o2048 | 32 | 16384 | 2048 | 398.63 | 640 | 0 | 1.61 | 26304.81 | 403.52 | 26708.32 | 1715.04 | 2963.25 | 13514.42 | 70.47 | 86.65 | 95.98 | 19175.98 | 37536.35 | 50504.81 | ⚠️ | +| c32_i16384_o256 | 32 | 16384 | 256 | 383.92 | 640 | 0 | 1.67 | 27312.05 | 348.16 | 27660.21 | 1826.81 | 3388.69 | 13517.47 | 80.79 | 92.93 | 101.63 | 18646.62 | 24405.54 | 29695.09 | ❌ | +| c32_i16384_o512 | 32 | 16384 | 512 | 393.37 | 640 | 0 | 1.63 | 26656.12 | 383.78 | 27039.90 | 1747.04 | 3400.82 | 13513.48 | 73.78 | 90.30 | 96.96 | 19000.89 | 36338.55 | 41160.95 | ❌ | +| c32_i2048_o1024 | 32 | 2048 | 1024 | 112.12 | 640 | 0 | 5.71 | 11690.22 | 1270.45 | 12960.67 | 246.53 | 373.10 | 1896.26 | 22.21 | 24.25 | 25.25 | 5165.11 | 8407.83 | 11262.12 | ✅ | +| c32_i2048_o128 | 32 | 2048 | 128 | 73.91 | 640 | 0 | 8.66 | 17733.36 | 1105.41 | 18838.77 | 904.68 | 1464.61 | 1844.76 | 20.43 | 26.38 | 27.06 | 3492.80 | 3746.52 | 4348.11 | ✅ | +| c32_i2048_o2048 | 32 | 2048 | 2048 | 111.79 | 640 | 0 | 5.73 | 11725.08 | 1282.42 | 13007.50 | 246.50 | 442.18 | 1847.04 | 22.17 | 24.28 | 25.62 | 5185.73 | 8810.37 | 10858.07 | ✅ | +| c32_i2048_o256 | 32 | 2048 | 256 | 103.61 | 640 | 0 | 6.18 | 12650.11 | 1258.11 | 13908.22 | 240.98 | 367.88 | 1848.70 | 22.80 | 24.74 | 25.47 | 4859.84 | 6298.45 | 6687.99 | ✅ | +| c32_i2048_o4096 | 32 | 2048 | 4096 | 106.84 | 640 | 0 | 5.99 | 12268.59 | 1282.15 | 13550.74 | 250.26 | 439.47 | 1837.79 | 22.45 | 24.65 | 25.84 | 5036.07 | 8060.13 | 10159.85 | ✅ | +| c32_i2048_o512 | 32 | 2048 | 512 | 111.06 | 640 | 0 | 5.76 | 11802.41 | 1281.65 | 13084.06 | 242.26 | 437.24 | 1893.17 | 22.15 | 24.51 | 25.55 | 5136.64 | 8816.12 | 11310.11 | ✅ | +| c32_i4096_o1024 | 32 | 4096 | 1024 | 141.80 | 640 | 0 | 4.51 | 18486.39 | 947.32 | 19433.71 | 432.85 | 813.49 | 3483.06 | 30.04 | 33.92 | 36.32 | 6702.72 | 11605.60 | 14590.75 | ✅ | +| c32_i4096_o128 | 32 | 4096 | 128 | 114.34 | 640 | 0 | 5.60 | 22925.90 | 710.98 | 23636.88 | 1053.81 | 1909.50 | 3485.20 | 35.10 | 41.98 | 43.26 | 5478.89 | 6106.50 | 7458.03 | ✅ | +| c32_i4096_o2048 | 32 | 4096 | 2048 | 144.45 | 640 | 0 | 4.43 | 18147.38 | 939.67 | 19087.05 | 434.39 | 747.31 | 3483.20 | 29.88 | 33.53 | 35.93 | 6700.04 | 10714.15 | 14733.26 | ✅ | +| c32_i4096_o256 | 32 | 4096 | 256 | 137.28 | 640 | 0 | 4.66 | 19095.92 | 921.51 | 20017.42 | 441.00 | 731.75 | 3478.68 | 30.98 | 34.42 | 36.58 | 6528.50 | 8638.60 | 10034.50 | ✅ | +| c32_i4096_o4096 | 32 | 4096 | 4096 | 143.01 | 640 | 0 | 4.48 | 18330.54 | 961.50 | 19292.04 | 429.61 | 749.16 | 3469.34 | 29.60 | 33.17 | 35.01 | 6743.49 | 10500.86 | 14281.37 | ✅ | +| c32_i4096_o512 | 32 | 4096 | 512 | 143.19 | 640 | 0 | 4.47 | 18307.18 | 929.29 | 19236.47 | 430.74 | 691.68 | 3532.95 | 30.57 | 34.15 | 36.96 | 6735.35 | 10401.62 | 14892.06 | ✅ | +| c32_i512_o256 | 32 | 512 | 256 | 98.73 | 640 | 0 | 6.48 | 3319.01 | 1330.82 | 4649.83 | 186.98 | 378.34 | 678.28 | 21.82 | 23.82 | 24.54 | 4649.84 | 6039.51 | 6163.96 | ✅ | +| c4_i512_o256 | 4 | 512 | 256 | 46.00 | 80 | 0 | 1.74 | 890.42 | 360.47 | 1250.89 | 164.64 | 347.47 | 374.55 | 8.92 | 10.51 | 11.09 | 1997.44 | 2523.64 | 2675.43 | ✅ | +| c64_i512_o256 | 64 | 512 | 256 | 140.30 | 1280 | 0 | 9.12 | 4671.07 | 1901.53 | 6572.60 | 229.06 | 453.78 | 1147.81 | 31.26 | 35.35 | 36.92 | 6712.71 | 8741.37 | 9237.35 | ✅ | +| c8_i1024_o1024 | 8 | 1024 | 1024 | 54.01 | 160 | 0 | 2.96 | 3033.30 | 621.77 | 3655.07 | 144.65 | 218.00 | 394.00 | 10.80 | 11.82 | 12.24 | 2395.48 | 3214.61 | 3506.80 | ✅ | +| c8_i1024_o128 | 8 | 1024 | 128 | 30.89 | 160 | 0 | 5.18 | 5304.66 | 662.82 | 5967.48 | 307.64 | 345.56 | 389.57 | 8.30 | 9.77 | 9.80 | 1360.75 | 1376.96 | 1412.43 | ✅ | +| c8_i1024_o2048 | 8 | 1024 | 2048 | 53.44 | 160 | 0 | 2.99 | 3065.61 | 627.08 | 3692.69 | 147.14 | 284.31 | 375.40 | 10.76 | 11.60 | 12.31 | 2391.87 | 3190.58 | 3558.55 | ✅ | +| c8_i1024_o256 | 8 | 1024 | 256 | 52.32 | 160 | 0 | 3.06 | 3131.63 | 620.55 | 3752.18 | 144.06 | 216.85 | 383.28 | 10.89 | 11.75 | 12.29 | 2345.43 | 2965.11 | 3042.16 | ✅ | +| c8_i1024_o4096 | 8 | 1024 | 4096 | 55.32 | 160 | 0 | 2.89 | 2961.58 | 621.49 | 3583.07 | 144.43 | 220.03 | 380.43 | 10.73 | 11.56 | 12.09 | 2438.42 | 3382.06 | 3756.38 | ✅ | +| c8_i1024_o512 | 8 | 1024 | 512 | 54.03 | 160 | 0 | 2.96 | 3032.14 | 632.21 | 3664.35 | 144.40 | 216.10 | 375.86 | 10.74 | 11.48 | 11.76 | 2425.82 | 3208.35 | 3616.77 | ✅ | +| c8_i16384_o1024 | 8 | 16384 | 1024 | 138.97 | 160 | 0 | 1.15 | 18863.47 | 287.59 | 19151.06 | 967.72 | 1600.40 | 3663.33 | 21.47 | 27.32 | 29.59 | 6298.76 | 12516.84 | 14588.60 | ✅ | +| c8_i16384_o128 | 8 | 16384 | 128 | 111.93 | 160 | 0 | 1.43 | 23420.44 | 181.33 | 23601.76 | 2556.64 | 4119.31 | 4136.10 | 20.74 | 33.97 | 34.00 | 5167.54 | 5417.75 | 5933.67 | ⚠️ | +| c8_i16384_o2048 | 8 | 16384 | 2048 | 136.26 | 160 | 0 | 1.17 | 19238.95 | 275.01 | 19513.96 | 961.94 | 1572.87 | 3673.56 | 22.52 | 28.43 | 30.68 | 6163.97 | 11684.40 | 14272.99 | ✅ | +| c8_i16384_o256 | 8 | 16384 | 256 | 127.82 | 160 | 0 | 1.25 | 20509.06 | 253.69 | 20762.74 | 1027.09 | 1938.52 | 3668.72 | 24.19 | 29.35 | 30.47 | 5890.64 | 7584.77 | 7970.70 | ✅ | +| c8_i16384_o512 | 8 | 16384 | 512 | 136.61 | 160 | 0 | 1.17 | 19188.83 | 286.09 | 19474.92 | 969.45 | 1904.35 | 3667.98 | 21.65 | 26.89 | 29.25 | 6250.80 | 10966.73 | 12572.72 | ✅ | +| c8_i2048_o1024 | 8 | 2048 | 1024 | 59.22 | 160 | 0 | 2.70 | 5533.53 | 606.82 | 6140.34 | 155.89 | 244.40 | 621.82 | 10.83 | 11.86 | 12.11 | 2572.18 | 4486.64 | 5823.30 | ✅ | +| c8_i2048_o128 | 8 | 2048 | 128 | 36.79 | 160 | 0 | 4.35 | 8907.76 | 554.37 | 9462.13 | 337.85 | 507.02 | 613.47 | 10.27 | 12.09 | 12.51 | 1636.65 | 1709.00 | 1726.45 | ✅ | +| c8_i2048_o2048 | 8 | 2048 | 2048 | 57.55 | 160 | 0 | 2.78 | 5693.86 | 626.97 | 6320.83 | 156.06 | 275.62 | 603.04 | 10.76 | 11.60 | 11.99 | 2571.87 | 4101.50 | 5201.48 | ✅ | +| c8_i2048_o256 | 8 | 2048 | 256 | 52.19 | 160 | 0 | 3.07 | 6279.15 | 604.00 | 6883.15 | 158.73 | 268.03 | 611.54 | 11.13 | 12.15 | 12.45 | 2334.26 | 3004.26 | 3124.59 | ✅ | +| c8_i2048_o4096 | 8 | 2048 | 4096 | 59.22 | 160 | 0 | 2.70 | 5533.71 | 614.83 | 6148.54 | 164.46 | 304.53 | 685.46 | 10.84 | 11.77 | 11.98 | 2624.68 | 4094.22 | 5661.19 | ✅ | +| c8_i2048_o512 | 8 | 2048 | 512 | 57.76 | 160 | 0 | 2.77 | 5672.85 | 605.80 | 6278.66 | 154.18 | 248.11 | 617.88 | 10.86 | 11.81 | 12.00 | 2521.32 | 3808.31 | 4671.03 | ✅ | +| c8_i4096_o1024 | 8 | 4096 | 1024 | 64.92 | 160 | 0 | 2.46 | 10094.27 | 518.13 | 10612.40 | 256.33 | 442.48 | 1016.15 | 12.36 | 13.77 | 14.46 | 2846.88 | 4602.44 | 5170.71 | ✅ | +| c8_i4096_o128 | 8 | 4096 | 128 | 46.47 | 160 | 0 | 3.44 | 14102.60 | 440.04 | 14542.64 | 707.03 | 1047.28 | 1071.50 | 10.92 | 15.03 | 15.06 | 2091.88 | 2145.56 | 2225.52 | ✅ | +| c8_i4096_o2048 | 8 | 4096 | 2048 | 66.29 | 160 | 0 | 2.41 | 9886.22 | 513.33 | 10399.56 | 253.25 | 423.32 | 1022.59 | 12.39 | 14.05 | 14.66 | 2865.36 | 4853.71 | 6028.24 | ✅ | +| c8_i4096_o256 | 8 | 4096 | 256 | 62.26 | 160 | 0 | 2.57 | 10526.27 | 516.50 | 11042.77 | 256.86 | 457.13 | 1013.55 | 12.63 | 14.05 | 14.57 | 2789.27 | 3625.36 | 3833.62 | ✅ | +| c8_i4096_o4096 | 8 | 4096 | 4096 | 64.28 | 160 | 0 | 2.49 | 10195.14 | 543.84 | 10738.99 | 248.19 | 462.57 | 1014.75 | 12.27 | 13.68 | 14.32 | 2921.13 | 5570.16 | 8112.52 | ✅ | +| c8_i4096_o512 | 8 | 4096 | 512 | 65.25 | 160 | 0 | 2.45 | 10043.48 | 538.74 | 10582.22 | 247.69 | 442.69 | 1015.54 | 12.25 | 13.71 | 14.06 | 2931.06 | 5124.89 | 6513.32 | ✅ | +| c8_i512_o256 | 8 | 512 | 256 | 52.73 | 160 | 0 | 3.03 | 1553.68 | 629.48 | 2183.16 | 135.86 | 271.88 | 289.07 | 10.80 | 11.89 | 12.07 | 2369.82 | 3039.64 | 3126.47 | ✅ | +| c128_i16384_o1024 | 128 | 16384 | 1024 | 1401.24 | 2560 | 0 | 1.83 | 29932.73 | 453.17 | 30385.90 | 6827.03 | 13719.55 | 53845.73 | 254.25 | 266.59 | 267.01 | 69085.30 | 135447.23 | 185844.63 | ❌ | +| c128_i16384_o2048 | 128 | 16384 | 2048 | 1400.62 | 2560 | 0 | 1.83 | 29945.97 | 468.45 | 30414.41 | 5874.33 | 13475.14 | 53934.11 | 249.38 | 266.26 | 266.71 | 69277.40 | 141505.20 | 189264.63 | ❌ | +| c1_i131072_o128 | 1 | 131072 | 128 | 182.19 | 20 | 0 | 0.11 | 14388.29 | 12.72 | 14401.01 | 5392.41 | 5419.10 | 5420.54 | 6.96 | 6.97 | 6.98 | 6192.49 | 6221.42 | 6221.43 | ⚠️ | +| c1_i131072_o256 | 1 | 131072 | 256 | 195.04 | 20 | 0 | 0.10 | 13440.21 | 19.01 | 13459.22 | 5397.54 | 5431.66 | 5440.22 | 7.00 | 7.03 | 7.03 | 6686.72 | 7103.29 | 7119.59 | ⚠️ | +| c1_i131072_o512 | 1 | 131072 | 512 | 201.48 | 20 | 0 | 0.10 | 13010.69 | 20.19 | 13030.87 | 5500.61 | 5555.72 | 5562.25 | 7.00 | 7.02 | 7.03 | 6917.34 | 7364.34 | 8109.96 | ⚠️ | +| c1_i262144_o256 | 1 | 262144 | 256 | 426.96 | 20 | 0 | 0.05 | 12279.65 | 8.19 | 12287.84 | 13218.70 | 13504.54 | 13586.50 | 7.22 | 7.32 | 7.36 | 14470.75 | 14834.79 | 14835.15 | ⚠️ | +| c1_i524288_o128 | 1 | 524288 | 128 | 1098.28 | 20 | 0 | 0.02 | 9547.46 | 1.62 | 9549.08 | 36239.34 | 36595.44 | 37102.25 | 9.37 | 9.80 | 37.00 | 36907.34 | 37308.40 | 37807.76 | ⚠️ | +| c1_i65536_o1024 | 1 | 65536 | 1024 | 114.83 | 20 | 0 | 0.17 | 11414.14 | 39.37 | 11453.51 | 2462.78 | 2475.26 | 2495.15 | 6.92 | 6.93 | 6.93 | 4020.21 | 5428.03 | 7058.64 | ✅ | +| c1_i65536_o128 | 1 | 65536 | 128 | 96.37 | 20 | 0 | 0.21 | 13600.71 | 24.89 | 13625.61 | 2497.10 | 2528.43 | 2558.93 | 6.87 | 6.87 | 6.88 | 3313.69 | 3344.68 | 3373.65 | ✅ | +| c1_i65536_o256 | 1 | 65536 | 256 | 105.96 | 20 | 0 | 0.19 | 12370.49 | 32.66 | 12403.15 | 2460.46 | 2472.33 | 2483.17 | 6.92 | 6.93 | 6.93 | 3650.21 | 4159.14 | 4159.75 | ✅ | +| c1_i65536_o512 | 1 | 65536 | 512 | 108.81 | 20 | 0 | 0.18 | 12045.94 | 34.57 | 12080.52 | 2461.82 | 2468.77 | 2498.35 | 6.92 | 6.93 | 6.93 | 3756.97 | 4315.50 | 4593.51 | ✅ | +| c32_i65536_o1024 | 32 | 65536 | 1024 | 395.45 | 160 | 0 | 0.40 | 26515.91 | 89.22 | 26605.13 | 18530.86 | 54451.46 | 68783.28 | 252.00 | 283.71 | 284.41 | 72731.85 | 117936.92 | 151338.20 | ❌ | +| c8_i131072_o512 | 8 | 131072 | 512 | 887.91 | 160 | 0 | 0.18 | 23619.01 | 33.13 | 23652.14 | 12820.61 | 25478.95 | 32945.62 | 156.62 | 223.19 | 294.51 | 41479.53 | 66809.84 | 79279.57 | ❌ | +| c8_i262144_o128 | 8 | 262144 | 128 | 2121.21 | 160 | 0 | 0.08 | 19773.16 | 9.48 | 19782.64 | 51391.65 | 61587.85 | 80661.75 | 382.04 | 388.68 | 389.24 | 99110.05 | 110875.08 | 129961.05 | ❌ | +| c8_i65536_o1024 | 8 | 65536 | 1024 | 422.41 | 160 | 0 | 0.38 | 24823.70 | 76.89 | 24900.59 | 4665.85 | 8696.15 | 14818.98 | 73.86 | 99.00 | 113.60 | 19699.15 | 35998.21 | 56237.93 | ❌ | +| c8_i65536_o128 | 8 | 65536 | 128 | 402.35 | 160 | 0 | 0.40 | 26061.00 | 46.48 | 26107.48 | 8659.91 | 13193.45 | 15765.71 | 87.87 | 143.78 | 166.11 | 18805.63 | 21397.73 | 22483.11 | ❌ | +| c8_i65536_o256 | 8 | 65536 | 256 | 416.75 | 160 | 0 | 0.38 | 25160.80 | 67.25 | 25228.06 | 5113.03 | 10698.95 | 14785.06 | 82.53 | 113.13 | 132.88 | 19445.90 | 27248.32 | 32918.89 | ❌ | +| c8_i65536_o512 | 8 | 65536 | 512 | 422.66 | 160 | 0 | 0.38 | 24809.03 | 76.23 | 24885.26 | 4532.43 | 9015.11 | 14789.72 | 75.30 | 108.22 | 126.16 | 19689.27 | 34072.42 | 42942.27 | ❌ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json new file mode 100644 index 0000000..7c20a16 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json @@ -0,0 +1,6428 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp8_custom_bench", + "run_id": "20260711-193756", + "timestamp": "2026-07-11T19:37:56+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "def1355", + "git_dirty": "dirty", + "description": "H200 1x vLLM TP=8 single-service benchmark using bench_client.py" + }, + "config": { + "tp": 8, + "num_services": 1, + "ports": "30005", + "gpu_groups": "0,1,2,3,4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "2 512 256 40", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [ + { + "name": "c128_i1024_o1024", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 206.74895269901026, + "request_throughput": 12.38216671272287, + "input_token_throughput": 12679.33871382822, + "output_token_throughput": 2588.491951294716, + "total_token_throughput": 15267.830665122934, + "total_input_tokens": 2621440, + "total_output_tokens": 535168, + "e2e_ms": { + "mean": 9995.612086433426, + "p50": 10042.748603475047, + "p90": 12876.024052471621, + "p95": 13713.982918864353, + "p99": 15284.945209203279 + }, + "ttft_ms": { + "mean": 355.5099342091353, + "p50": 243.9829669892788, + "p90": 386.2356346042361, + "p95": 540.9974577196409, + "p99": 3399.252886212198 + }, + "tpot_ms": { + "mean": 46.34608583039063, + "p50": 47.0857676038808, + "p90": 49.548431669654995, + "p95": 50.57579517885789, + "p99": 52.541381539005876 + }, + "itl_ms": { + "mean": 46.11291511528496, + "p50": 46.881382113519905, + "p90": 49.288727774137215, + "p95": 50.29987956637689, + "p99": 52.29071026166347 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 133.00950148300035, + "request_throughput": 19.246745318620626, + "input_token_throughput": 19708.66720626752, + "output_token_throughput": 2459.6964604202653, + "total_token_throughput": 22168.363666687786, + "total_input_tokens": 2621440, + "total_output_tokens": 327163, + "e2e_ms": { + "mean": 6458.710522773936, + "p50": 6450.677961984184, + "p90": 6679.613613686524, + "p95": 7012.215024293982, + "p99": 8759.515378992073 + }, + "ttft_ms": { + "mean": 1183.3169419023761, + "p50": 1213.7640539731365, + "p90": 1461.3176107173786, + "p95": 1479.2902125336695, + "p99": 3426.818067198619 + }, + "tpot_ms": { + "mean": 41.60377345263331, + "p50": 41.42248976997854, + "p90": 45.77268147799847, + "p95": 47.49906829926808, + "p99": 49.21793915747983 + }, + "itl_ms": { + "mean": 41.60175583400803, + "p50": 41.42150920233689, + "p90": 45.77246147393997, + "p95": 47.49885068504657, + "p99": 49.21768283782669 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_128.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 208.80468665598892, + "request_throughput": 12.260261208684772, + "input_token_throughput": 12554.507477693207, + "output_token_throughput": 2547.1602602304197, + "total_token_throughput": 15101.667737923628, + "total_input_tokens": 2621440, + "total_output_tokens": 531859, + "e2e_ms": { + "mean": 10106.575596190418, + "p50": 10063.31227402552, + "p90": 13078.549599891994, + "p95": 13980.859304094341, + "p99": 15725.898965689119 + }, + "ttft_ms": { + "mean": 351.4643375147216, + "p50": 240.56834846851416, + "p90": 374.20819198014215, + "p95": 493.9341375400565, + "p99": 3429.205895262421 + }, + "tpot_ms": { + "mean": 47.17323325059421, + "p50": 47.75218639386304, + "p90": 49.633491854371265, + "p95": 52.36074024377563, + "p99": 59.290788935476655 + }, + "itl_ms": { + "mean": 46.937751244996235, + "p50": 47.51707957709188, + "p90": 49.415206330399855, + "p95": 51.937400787877415, + "p99": 59.03572046826236 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 205.11042719194666, + "request_throughput": 12.481081703390428, + "input_token_throughput": 12780.627664271798, + "output_token_throughput": 2561.2349756766853, + "total_token_throughput": 15341.862639948484, + "total_input_tokens": 2621440, + "total_output_tokens": 525336, + "e2e_ms": { + "mean": 9935.310300847526, + "p50": 10115.252760995645, + "p90": 12385.100717213936, + "p95": 12606.547662557568, + "p99": 13028.91704150126 + }, + "ttft_ms": { + "mean": 351.2175043693105, + "p50": 243.14397198031656, + "p90": 373.0044780881144, + "p95": 456.94061022368277, + "p99": 3426.5382378635695 + }, + "tpot_ms": { + "mean": 46.93347030558093, + "p50": 47.9216824371189, + "p90": 49.75475172500139, + "p95": 50.35507099599905, + "p99": 51.38505285855793 + }, + "itl_ms": { + "mean": 46.72323950761121, + "p50": 47.723609778449614, + "p90": 49.567892764403894, + "p95": 50.12071192187135, + "p99": 51.17095904803956 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_256.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 208.6310452229809, + "request_throughput": 12.270465295631917, + "input_token_throughput": 12564.956462727083, + "output_token_throughput": 2562.1786030390786, + "total_token_throughput": 15127.13506576616, + "total_input_tokens": 2621440, + "total_output_tokens": 534550, + "e2e_ms": { + "mean": 10079.333489348573, + "p50": 10091.16335248109, + "p90": 12926.991533790713, + "p95": 13938.759127602681, + "p99": 15626.708482055688 + }, + "ttft_ms": { + "mean": 352.13409095886163, + "p50": 244.19353948906064, + "p90": 370.4628453939222, + "p95": 475.0491648941544, + "p99": 3401.064061677898 + }, + "tpot_ms": { + "mean": 46.82233196854027, + "p50": 47.89751414392755, + "p90": 49.819039048612304, + "p95": 50.3518367372333, + "p99": 51.5907629143145 + }, + "itl_ms": { + "mean": 46.58335775413696, + "p50": 47.644176474945255, + "p90": 49.59239070185347, + "p95": 50.0927766120728, + "p99": 51.20594759062769 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 210.2132963129552, + "request_throughput": 12.178106927112726, + "input_token_throughput": 12470.381493363431, + "output_token_throughput": 2534.3829783576184, + "total_token_throughput": 15004.76447172105, + "total_input_tokens": 2621440, + "total_output_tokens": 532761, + "e2e_ms": { + "mean": 10138.544655314876, + "p50": 10175.040662987158, + "p90": 13069.769149302738, + "p95": 13931.679938701564, + "p99": 15398.916205506535 + }, + "ttft_ms": { + "mean": 355.65935904862727, + "p50": 240.44733747723512, + "p90": 393.7936486501712, + "p95": 519.455098820618, + "p99": 3397.328862186987 + }, + "tpot_ms": { + "mean": 47.251763059437835, + "p50": 47.96887544839744, + "p90": 50.820445244109074, + "p95": 52.83991169163644, + "p99": 55.04587439127752 + }, + "itl_ms": { + "mean": 47.00573781106416, + "p50": 47.71712084654526, + "p90": 50.64945889698164, + "p95": 52.655936409592336, + "p99": 54.68855691765819 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_512.jsonl" + }, + { + "name": "c128_i16384_o128", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1359.066230058961, + "request_throughput": 1.883646244295937, + "input_token_throughput": 30861.66006654463, + "output_token_throughput": 240.90584605710933, + "total_token_throughput": 31102.56591260174, + "total_input_tokens": 41943040, + "total_output_tokens": 327407, + "e2e_ms": { + "mean": 67037.50906514256, + "p50": 67357.02024202328, + "p90": 67636.39427277376, + "p95": 67882.99461098795, + "p99": 86482.76843836233 + }, + "ttft_ms": { + "mean": 34308.65381123959, + "p50": 34204.28299551713, + "p90": 34482.4229988968, + "p95": 34726.7470557912, + "p99": 53850.1370839576 + }, + "tpot_ms": { + "mean": 257.92863620986134, + "p50": 261.05848694491516, + "p90": 261.1143110553719, + "p95": 261.1287856930326, + "p99": 261.24581678764065 + }, + "itl_ms": { + "mean": 257.9097102152317, + "p50": 261.05787166541876, + "p90": 261.1123652793247, + "p95": 261.1257119797258, + "p99": 261.14682886486884 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_16384_128.jsonl" + }, + { + "name": "c128_i16384_o256", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1383.2075693949591, + "request_throughput": 1.850770670030234, + "input_token_throughput": 30323.026657775354, + "output_token_throughput": 393.1159805884024, + "total_token_throughput": 30716.14263836376, + "total_input_tokens": 41943040, + "total_output_tokens": 543761, + "e2e_ms": { + "mean": 68522.27549761867, + "p50": 68875.66710749525, + "p90": 80996.00835689344, + "p95": 82348.9028138516, + "p99": 113143.29638896219 + }, + "ttft_ms": { + "mean": 13674.308820916713, + "p50": 12427.863965480356, + "p90": 15051.10284507391, + "p95": 16619.003294425784, + "p99": 53959.96751072168 + }, + "tpot_ms": { + "mean": 259.7002911860435, + "p50": 265.0982799215538, + "p90": 265.59337430180994, + "p95": 265.74050924260996, + "p99": 266.01097282252385 + }, + "itl_ms": { + "mean": 258.7413562117342, + "p50": 263.88649323717783, + "p90": 264.0145869704262, + "p95": 264.049955442955, + "p99": 264.0919490046659 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_16384_256.jsonl" + }, + { + "name": "c128_i16384_o512", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1394.5807457869523, + "request_throughput": 1.8356771436389003, + "input_token_throughput": 30075.734321379743, + "output_token_throughput": 449.1364174446214, + "total_token_throughput": 30524.870738824364, + "total_input_tokens": 41943040, + "total_output_tokens": 626357, + "e2e_ms": { + "mean": 68997.86024277919, + "p50": 61218.58670547954, + "p90": 111877.90824500262, + "p95": 134484.1842920287, + "p99": 142208.8375359383 + }, + "ttft_ms": { + "mean": 6929.190793514613, + "p50": 5308.056419016793, + "p90": 8465.563770505833, + "p95": 14006.295557459864, + "p99": 53929.18027237865 + }, + "tpot_ms": { + "mean": 256.42045890030613, + "p50": 265.1347581875615, + "p90": 266.20946280775973, + "p95": 266.3879577864928, + "p99": 266.67562587471275 + }, + "itl_ms": { + "mean": 255.26895726498742, + "p50": 264.1093867197653, + "p90": 264.64687804993525, + "p95": 264.6946616676973, + "p99": 264.82593538929 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_16384_512.jsonl" + }, + { + "name": "c128_i2048_o1024", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 274.182873593003, + "request_throughput": 9.336834086143773, + "input_token_throughput": 19121.836208422446, + "output_token_throughput": 2068.13428048655, + "total_token_throughput": 21189.970488908995, + "total_input_tokens": 5242880, + "total_output_tokens": 567047, + "e2e_ms": { + "mean": 13253.984380322005, + "p50": 12231.172265019268, + "p90": 18871.34518827661, + "p95": 21801.421404152643, + "p99": 28048.55546446866 + }, + "ttft_ms": { + "mean": 541.546650899204, + "p50": 335.68961051059887, + "p90": 544.6287516097073, + "p95": 764.6706944942717, + "p99": 6604.665919882827 + }, + "tpot_ms": { + "mean": 57.69224896148207, + "p50": 58.662424903217726, + "p90": 61.13828027834164, + "p95": 62.17687225050866, + "p99": 65.53232212942441 + }, + "itl_ms": { + "mean": 57.41329839820827, + "p50": 58.421607822445225, + "p90": 60.78241045852147, + "p95": 61.94631333915732, + "p99": 65.1391069201185 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_1024.jsonl" + }, + { + "name": "c128_i2048_o128", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 212.99189165403368, + "request_throughput": 12.019236883243664, + "input_token_throughput": 24615.397136883024, + "output_token_throughput": 1530.0206851504734, + "total_token_throughput": 26145.417822033498, + "total_input_tokens": 5242880, + "total_output_tokens": 325882, + "e2e_ms": { + "mean": 10431.321765192843, + "p50": 10470.64316298929, + "p90": 11085.870366205927, + "p95": 11370.95141907048, + "p99": 15884.139941605354 + }, + "ttft_ms": { + "mean": 1293.090087216433, + "p50": 1246.8626230256632, + "p90": 1747.983089205809, + "p95": 1997.4948156188475, + "p99": 6703.310411417624 + }, + "tpot_ms": { + "mean": 72.32697406863772, + "p50": 73.10035818496601, + "p90": 78.97615628596579, + "p95": 80.38428567705573, + "p99": 81.7834753297109 + }, + "itl_ms": { + "mean": 72.29165904992381, + "p50": 73.09435472038106, + "p90": 78.97258139450074, + "p95": 80.38391650942664, + "p99": 81.7239847918815 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_128.jsonl" + }, + { + "name": "c128_i2048_o2048", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 272.0649238389451, + "request_throughput": 9.409518742355221, + "input_token_throughput": 19270.694384343493, + "output_token_throughput": 2055.1601879079185, + "total_token_throughput": 21325.854572251414, + "total_input_tokens": 5242880, + "total_output_tokens": 559137, + "e2e_ms": { + "mean": 13143.514178946545, + "p50": 12082.19302649377, + "p90": 18679.455636080816, + "p95": 21512.404798675576, + "p99": 28583.361954419513 + }, + "ttft_ms": { + "mean": 542.6618820346221, + "p50": 339.305431523826, + "p90": 539.0634975978171, + "p95": 696.4638119126903, + "p99": 6595.59385088447 + }, + "tpot_ms": { + "mean": 58.015877285501915, + "p50": 59.11393804880664, + "p90": 61.71648090264369, + "p95": 62.670298319004864, + "p99": 65.22564691015543 + }, + "itl_ms": { + "mean": 57.71674817147152, + "p50": 58.78867086320406, + "p90": 61.42682309865147, + "p95": 62.303509328181136, + "p99": 64.79217375997129 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_2048.jsonl" + }, + { + "name": "c128_i2048_o256", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 263.82883767504245, + "request_throughput": 9.70326072979614, + "input_token_throughput": 19872.277974622495, + "output_token_throughput": 1986.560698287076, + "total_token_throughput": 21858.838672909573, + "total_input_tokens": 5242880, + "total_output_tokens": 524112, + "e2e_ms": { + "mean": 12841.303998676336, + "p50": 12770.766553498106, + "p90": 16183.979681675555, + "p95": 16454.99841661367, + "p99": 19770.974340796816 + }, + "ttft_ms": { + "mean": 552.7428740571622, + "p50": 338.345098978607, + "p90": 574.8256633232812, + "p95": 766.9361727079388, + "p99": 6667.41566293058 + }, + "tpot_ms": { + "mean": 60.36128601226305, + "p50": 61.53765959324275, + "p90": 64.03844464794237, + "p95": 64.80210034227288, + "p99": 67.6115024987387 + }, + "itl_ms": { + "mean": 60.09187703963762, + "p50": 61.29940147707798, + "p90": 63.74183259078168, + "p95": 64.42361930701372, + "p99": 67.1758936519173 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_256.jsonl" + }, + { + "name": "c128_i2048_o4096", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 274.1830678950064, + "request_throughput": 9.336827469522325, + "input_token_throughput": 19121.82265758172, + "output_token_throughput": 2075.5913352677335, + "total_token_throughput": 21197.41399284945, + "total_input_tokens": 5242880, + "total_output_tokens": 569092, + "e2e_ms": { + "mean": 13271.956057284842, + "p50": 12218.301199522102, + "p90": 19085.673499788387, + "p95": 22132.755441660993, + "p99": 28679.850054419945 + }, + "ttft_ms": { + "mean": 538.8494964908205, + "p50": 329.8501919780392, + "p90": 536.288884928217, + "p95": 772.7042476995854, + "p99": 6666.243869229802 + }, + "tpot_ms": { + "mean": 57.59475360987989, + "p50": 58.62241071854672, + "p90": 60.865193869045825, + "p95": 61.732659389700814, + "p99": 65.98072567714664 + }, + "itl_ms": { + "mean": 57.31168219239646, + "p50": 58.35251099982605, + "p90": 60.52328757509878, + "p95": 61.41718757690342, + "p99": 65.77234831641486 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_4096.jsonl" + }, + { + "name": "c128_i2048_o512", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 271.79637533798814, + "request_throughput": 9.418815820544156, + "input_token_throughput": 19289.73480047443, + "output_token_throughput": 2074.255034854407, + "total_token_throughput": 21363.989835328837, + "total_input_tokens": 5242880, + "total_output_tokens": 563775, + "e2e_ms": { + "mean": 13185.11682162771, + "p50": 12167.376212484669, + "p90": 18797.917102725478, + "p95": 21822.20230518142, + "p99": 28335.43754096433 + }, + "ttft_ms": { + "mean": 544.2361912605747, + "p50": 335.83095151698217, + "p90": 560.1696573663503, + "p95": 780.7416238734743, + "p99": 6725.671748666791 + }, + "tpot_ms": { + "mean": 57.75487659402953, + "p50": 58.80900923345935, + "p90": 60.99471152181436, + "p95": 61.62449590523302, + "p99": 65.39932975583538 + }, + "itl_ms": { + "mean": 57.481931975649466, + "p50": 58.51227060084552, + "p90": 60.6782899592584, + "p95": 61.29368894317035, + "p99": 65.19139830356515 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_512.jsonl" + }, + { + "name": "c128_i4096_o1024", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 424.82515185599914, + "request_throughput": 6.026008556263049, + "input_token_throughput": 24682.53104645345, + "output_token_throughput": 1273.8087602295016, + "total_token_throughput": 25956.33980668295, + "total_input_tokens": 10485760, + "total_output_tokens": 541146, + "e2e_ms": { + "mean": 20697.6679702115, + "p50": 19206.427480472485, + "p90": 29670.110822987044, + "p95": 35507.95289489034, + "p99": 47170.14222518534 + }, + "ttft_ms": { + "mean": 1014.2606450839367, + "p50": 589.3256725103129, + "p90": 1090.3332160029092, + "p95": 1488.9069345052123, + "p99": 13325.899195196107 + }, + "tpot_ms": { + "mean": 93.93489520852809, + "p50": 95.33726487295453, + "p90": 101.10578242438245, + "p95": 103.74371460626769, + "p99": 115.23677950752673 + }, + "itl_ms": { + "mean": 93.4331025955482, + "p50": 94.93783975657502, + "p90": 100.51129917148953, + "p95": 103.0640243406409, + "p99": 114.33476458414724 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_1024.jsonl" + }, + { + "name": "c128_i4096_o128", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 371.4178167580394, + "request_throughput": 6.892507264043596, + "input_token_throughput": 28231.70975352257, + "output_token_throughput": 873.7060672870264, + "total_token_throughput": 29105.415820809598, + "total_input_tokens": 10485760, + "total_output_tokens": 324510, + "e2e_ms": { + "mean": 18310.57876672269, + "p50": 18448.31012902432, + "p90": 19225.236213795142, + "p95": 19462.368021774455, + "p99": 30169.283645756623 + }, + "ttft_ms": { + "mean": 1687.7391880757386, + "p50": 1529.599169472931, + "p90": 2042.0844946987927, + "p95": 2056.5731700480683, + "p99": 13344.424836942344 + }, + "tpot_ms": { + "mean": 132.0821975364293, + "p50": 133.9608105825348, + "p90": 140.01268617562238, + "p95": 142.1111826331164, + "p99": 145.57791828146406 + }, + "itl_ms": { + "mean": 131.99716351450823, + "p50": 133.95380003535462, + "p90": 140.00703786379717, + "p95": 142.11082250321206, + "p99": 145.07502242550618 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_128.jsonl" + }, + { + "name": "c128_i4096_o2048", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 425.2518261469668, + "request_throughput": 6.019962390744127, + "input_token_throughput": 24657.765952487945, + "output_token_throughput": 1288.0344452905458, + "total_token_throughput": 25945.80039777849, + "total_input_tokens": 10485760, + "total_output_tokens": 547739, + "e2e_ms": { + "mean": 20779.354470113663, + "p50": 19269.694546994288, + "p90": 29949.315433588345, + "p95": 36358.763673005196, + "p99": 50447.49927988618 + }, + "ttft_ms": { + "mean": 1019.4579501709768, + "p50": 598.1724759622011, + "p90": 1096.0077470983379, + "p95": 1464.8891508928511, + "p99": 13332.857389408746 + }, + "tpot_ms": { + "mean": 93.12786586783068, + "p50": 94.68724737436233, + "p90": 99.93479914307919, + "p95": 101.5232748251677, + "p99": 111.69156290842999 + }, + "itl_ms": { + "mean": 92.65871392345448, + "p50": 94.17968940964437, + "p90": 99.46187302676536, + "p95": 101.09497997720692, + "p99": 110.93734985374068 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_2048.jsonl" + }, + { + "name": "c128_i4096_o256", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 413.8003815539996, + "request_throughput": 6.186557852813213, + "input_token_throughput": 25340.14096512292, + "output_token_throughput": 1217.7924972121846, + "total_token_throughput": 26557.933462335102, + "total_input_tokens": 10485760, + "total_output_tokens": 503923, + "e2e_ms": { + "mean": 20311.651262024407, + "p50": 20182.22546146717, + "p90": 26356.96079552872, + "p95": 26976.595176249975, + "p99": 33054.57394192112 + }, + "ttft_ms": { + "mean": 1065.7509551069552, + "p50": 660.4246550123207, + "p90": 1217.6136810914613, + "p95": 1554.3095287983313, + "p99": 13333.887037970126 + }, + "tpot_ms": { + "mean": 98.37467601072895, + "p50": 100.35475919842096, + "p90": 106.18230888031277, + "p95": 108.53097404087819, + "p99": 114.97146971749977 + }, + "itl_ms": { + "mean": 97.90174442685736, + "p50": 100.01569229517985, + "p90": 105.46145688671882, + "p95": 107.86138331901421, + "p99": 114.5162470124037 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_256.jsonl" + }, + { + "name": "c128_i4096_o4096", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 423.761194480001, + "request_throughput": 6.041138342413316, + "input_token_throughput": 24744.50265052494, + "output_token_throughput": 1282.561987930327, + "total_token_throughput": 26027.064638455267, + "total_input_tokens": 10485760, + "total_output_tokens": 543500, + "e2e_ms": { + "mean": 20723.11629090143, + "p50": 19256.915692501934, + "p90": 29976.309534144817, + "p95": 35255.544051312616, + "p99": 48374.307467846615 + }, + "ttft_ms": { + "mean": 1040.8386572691825, + "p50": 606.4721180300694, + "p90": 1181.8239250802435, + "p95": 1941.310445405544, + "p99": 13328.937759977998 + }, + "tpot_ms": { + "mean": 93.33886441284803, + "p50": 94.96167027747198, + "p90": 99.94806040223267, + "p95": 101.80580796030549, + "p99": 111.130208978832 + }, + "itl_ms": { + "mean": 92.84261079498634, + "p50": 94.58125016055442, + "p90": 99.3758299306611, + "p95": 101.0630073622523, + "p99": 110.63285175462228 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_4096.jsonl" + }, + { + "name": "c128_i4096_o512", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 423.6984583950252, + "request_throughput": 6.042032840282946, + "input_token_throughput": 24748.166513798948, + "output_token_throughput": 1274.4157768366804, + "total_token_throughput": 26022.58229063563, + "total_input_tokens": 10485760, + "total_output_tokens": 539968, + "e2e_ms": { + "mean": 20669.59312196584, + "p50": 19350.259740982438, + "p90": 29572.45529227657, + "p95": 35396.549672255096, + "p99": 49034.77441504888 + }, + "ttft_ms": { + "mean": 1032.5226390652233, + "p50": 598.7296614912339, + "p90": 1154.9013265117537, + "p95": 1562.556813692209, + "p99": 13323.3382030559 + }, + "tpot_ms": { + "mean": 93.86471050008637, + "p50": 95.59369101226693, + "p90": 101.53613190663701, + "p95": 103.68156125609973, + "p99": 112.50945685705346 + }, + "itl_ms": { + "mean": 93.36493755809302, + "p50": 95.09812282953814, + "p90": 100.97701147581158, + "p95": 103.27911349553783, + "p99": 111.96191368855567 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_512.jsonl" + }, + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 198.04905138298636, + "request_throughput": 12.926090693812432, + "input_token_throughput": 6618.158435231965, + "output_token_throughput": 2673.832549573583, + "total_token_throughput": 9291.990984805549, + "total_input_tokens": 1310720, + "total_output_tokens": 529550, + "e2e_ms": { + "mean": 9585.923150789631, + "p50": 10002.822702517733, + "p90": 12147.832186403684, + "p95": 13135.335343127372, + "p99": 14237.255100695762 + }, + "ttft_ms": { + "mean": 290.6632589968467, + "p50": 214.245242474135, + "p90": 345.00691113644274, + "p95": 500.99967721907893, + "p99": 1944.8683028796222 + }, + "tpot_ms": { + "mean": 45.15565344448567, + "p50": 44.83139648013622, + "p90": 51.02216221175303, + "p95": 52.67646848178537, + "p99": 57.18692985139731 + }, + "itl_ms": { + "mean": 44.973391207111554, + "p50": 44.674817022825394, + "p90": 50.939215799513484, + "p95": 52.50806826338534, + "p99": 56.95803494225427 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_512_256.jsonl" + }, + { + "name": "c16_i512_o256", + "config": { + "concurrency": 16, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 70.00084047595738, + "request_throughput": 4.5713736838618075, + "input_token_throughput": 2340.5433261372455, + "output_token_throughput": 954.2456854206282, + "total_token_throughput": 3294.789011557874, + "total_input_tokens": 163840, + "total_output_tokens": 66798, + "e2e_ms": { + "mean": 3227.912786142042, + "p50": 3303.816789004486, + "p90": 4077.1741356002167, + "p95": 4110.177500350983, + "p99": 4253.23702479538 + }, + "ttft_ms": { + "mean": 155.9941838546365, + "p50": 120.02038149512373, + "p90": 209.794121485902, + "p95": 299.16939962713525, + "p99": 421.0051234619459 + }, + "tpot_ms": { + "mean": 14.771496620725333, + "p50": 14.982428883652467, + "p90": 16.07892679411306, + "p95": 16.431581466333803, + "p99": 17.43982213951822 + }, + "itl_ms": { + "mean": 14.709423569418314, + "p50": 14.96633889086642, + "p90": 15.991478397076191, + "p95": 16.343193537253118, + "p99": 17.36775174975628 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_16_512_256.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 41.001702999987174, + "request_throughput": 0.48778461714154303, + "input_token_throughput": 499.49144795294006, + "output_token_throughput": 102.02015267515372, + "total_token_throughput": 601.5116006280938, + "total_input_tokens": 20480, + "total_output_tokens": 4183, + "e2e_ms": { + "mean": 1513.646431043162, + "p50": 1574.0484629932325, + "p90": 1819.820330955554, + "p95": 1949.2676253576064, + "p99": 1998.6603202793049 + }, + "ttft_ms": { + "mean": 113.2725309493253, + "p50": 111.85279348865151, + "p90": 113.1193755834829, + "p95": 116.29407556611115, + "p99": 137.7853335009422 + }, + "tpot_ms": { + "mean": 6.728842661121109, + "p50": 6.728117721421029, + "p90": 6.73756308164125, + "p95": 6.738021995986233, + "p99": 6.7388486460154216 + }, + "itl_ms": { + "mean": 6.692791953543756, + "p50": 6.693387171586178, + "p90": 6.6951132749056494, + "p95": 6.695167897476331, + "p99": 6.695272520098379 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o128", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 26.62190206797095, + "request_throughput": 0.7512611213479814, + "input_token_throughput": 769.291388260333, + "output_token_throughput": 96.16142353254162, + "total_token_throughput": 865.4528117928745, + "total_input_tokens": 20480, + "total_output_tokens": 2560, + "e2e_ms": { + "mean": 961.8738458957523, + "p50": 961.338467488531, + "p90": 962.9271580080967, + "p95": 964.1112041892484, + "p99": 977.0850000297651 + }, + "ttft_ms": { + "mean": 112.09932600322645, + "p50": 111.56318802386522, + "p90": 112.80985138146207, + "p95": 114.09245460527019, + "p99": 127.54729093285275 + }, + "tpot_ms": { + "mean": 6.691137951909653, + "p50": 6.690667098384409, + "p90": 6.694461964299961, + "p95": 6.695022723819936, + "p99": 6.696434283411812 + }, + "itl_ms": { + "mean": 6.6905939176735485, + "p50": 6.69014484252172, + "p90": 6.6939145778067495, + "p95": 6.694489817490698, + "p99": 6.695905075294853 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_128.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 40.37348232802469, + "request_throughput": 0.49537465798726205, + "input_token_throughput": 507.26364977895634, + "output_token_throughput": 100.38767444111865, + "total_token_throughput": 607.651324220075, + "total_input_tokens": 20480, + "total_output_tokens": 4053, + "e2e_ms": { + "mean": 1470.1031726581277, + "p50": 1538.4048255218659, + "p90": 1877.3818529967684, + "p95": 1949.8724324192156, + "p99": 2093.403241672204 + }, + "ttft_ms": { + "mean": 113.53871540632099, + "p50": 112.32990000280552, + "p90": 115.95370600116439, + "p95": 118.16973473469263, + "p99": 129.37863651022778 + }, + "tpot_ms": { + "mean": 6.729301792871088, + "p50": 6.726905772857237, + "p90": 6.7408030649588095, + "p95": 6.749353885298162, + "p99": 6.749622846797336 + }, + "itl_ms": { + "mean": 6.6908831387070915, + "p50": 6.6918520466231985, + "p90": 6.694045093510076, + "p95": 6.694208334277694, + "p99": 6.694287661540632 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.66181249899091, + "request_throughput": 0.5042633893876849, + "input_token_throughput": 516.3657107329893, + "output_token_throughput": 101.40736760586343, + "total_token_throughput": 617.7730783388527, + "total_input_tokens": 20480, + "total_output_tokens": 4022, + "e2e_ms": { + "mean": 1457.720083554159, + "p50": 1424.1828475205693, + "p90": 1692.0858449302616, + "p95": 1819.5981201744871, + "p99": 1819.9626496067503 + }, + "ttft_ms": { + "mean": 112.03125940228347, + "p50": 111.31371150258929, + "p90": 112.13053134270012, + "p95": 113.19355736195577, + "p99": 128.9370090694865 + }, + "tpot_ms": { + "mean": 6.72637613476324, + "p50": 6.728691771932777, + "p90": 6.733196039297692, + "p95": 6.738730504584789, + "p99": 6.738919628883542 + }, + "itl_ms": { + "mean": 6.69216179570252, + "p50": 6.692514737256101, + "p90": 6.694407104034717, + "p95": 6.6958281427836415, + "p99": 6.697055611373656 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_256.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.069976897968445, + "request_throughput": 0.511902017557629, + "input_token_throughput": 524.1876659790121, + "output_token_throughput": 100.38398564305103, + "total_token_throughput": 624.5716516220631, + "total_input_tokens": 20480, + "total_output_tokens": 3922, + "e2e_ms": { + "mean": 1439.5377619453939, + "p50": 1474.0643894765526, + "p90": 1734.9704635154922, + "p95": 1773.7776646856219, + "p99": 1831.833771346137 + }, + "ttft_ms": { + "mean": 126.45561884564813, + "p50": 112.52622748725116, + "p90": 176.61038206424564, + "p95": 178.17131999472622, + "p99": 178.35113599256147 + }, + "tpot_ms": { + "mean": 6.731382327839597, + "p50": 6.729523633618532, + "p90": 6.741167108195672, + "p95": 6.742111598489635, + "p99": 6.7473587002338045 + }, + "itl_ms": { + "mean": 6.693141582456953, + "p50": 6.693129204125723, + "p90": 6.698133283139748, + "p95": 6.698879551499814, + "p99": 6.6989547085102945 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_4096.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 38.85685831902083, + "request_throughput": 0.5147096514030265, + "input_token_throughput": 527.0626830366991, + "output_token_throughput": 98.61836920881987, + "total_token_throughput": 625.6810522455189, + "total_input_tokens": 20480, + "total_output_tokens": 3832, + "e2e_ms": { + "mean": 1396.0474013991188, + "p50": 1421.305021009175, + "p90": 1761.8963810731657, + "p95": 1797.4548358935863, + "p99": 1820.3903575986624 + }, + "ttft_ms": { + "mean": 113.08509785449132, + "p50": 111.65635052020662, + "p90": 112.92990780784748, + "p95": 114.42024928692264, + "p99": 136.00369145569854 + }, + "tpot_ms": { + "mean": 6.7326943598063, + "p50": 6.7306789376316285, + "p90": 6.741523098781185, + "p95": 6.742240715060803, + "p99": 6.747612401371304 + }, + "itl_ms": { + "mean": 6.692899736605012, + "p50": 6.6925738048090775, + "p90": 6.695661536070422, + "p95": 6.695736518938616, + "p99": 6.696398640652917 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 60.01604464603588, + "request_throughput": 0.33324422024071226, + "input_token_throughput": 5459.87330442383, + "output_token_throughput": 74.28013669165476, + "total_token_throughput": 5534.153441115484, + "total_input_tokens": 327680, + "total_output_tokens": 4458, + "e2e_ms": { + "mean": 2136.322955050855, + "p50": 1987.1548090013675, + "p90": 2866.87489899341, + "p95": 2962.6648157864115, + "p99": 3211.9196255604033 + }, + "ttft_ms": { + "mean": 621.8708320520818, + "p50": 618.9937324961647, + "p90": 631.5798229188658, + "p95": 633.1030654429924, + "p99": 637.2123010957148 + }, + "tpot_ms": { + "mean": 6.824606008095292, + "p50": 6.822637892302804, + "p90": 6.828144618603244, + "p95": 6.829834324474402, + "p99": 6.849728735808008 + }, + "itl_ms": { + "mean": 6.7858919799987145, + "p50": 6.784267973909232, + "p90": 6.802818123600203, + "p95": 6.806531345537709, + "p99": 6.810954561452206 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 41.67448142397916, + "request_throughput": 0.47990999087734687, + "input_token_throughput": 7862.845290534451, + "output_token_throughput": 60.87658234279145, + "total_token_throughput": 7923.721872877242, + "total_input_tokens": 327680, + "total_output_tokens": 2537, + "e2e_ms": { + "mean": 1480.094525500317, + "p50": 1480.2975574857555, + "p90": 1482.2705925849732, + "p95": 1482.5216800556518, + "p99": 1482.9828344029374 + }, + "ttft_ms": { + "mean": 628.48725815129, + "p50": 628.42066999292, + "p90": 630.9192391112447, + "p95": 631.5766573417932, + "p99": 636.3627922814339 + }, + "tpot_ms": { + "mean": 6.766875583214485, + "p50": 6.762159404590623, + "p90": 6.772095540245729, + "p95": 6.806862495630048, + "p99": 6.809469593646936 + }, + "itl_ms": { + "mean": 6.766333328781184, + "p50": 6.76163371018548, + "p90": 6.771564252188961, + "p95": 6.8063165623694655, + "p99": 6.808896829262376 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_128.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 63.797216865001246, + "request_throughput": 0.3134932992817101, + "input_token_throughput": 5136.274215431538, + "output_token_throughput": 80.75587389496853, + "total_token_throughput": 5217.030089326508, + "total_input_tokens": 327680, + "total_output_tokens": 5152, + "e2e_ms": { + "mean": 2381.452842548606, + "p50": 2055.640481004957, + "p90": 3254.1592128574866, + "p95": 4577.4711255711745, + "p99": 5209.664837909512 + }, + "ttft_ms": { + "mean": 630.1534300495405, + "p50": 629.4059840147384, + "p90": 633.5870493319817, + "p95": 636.3642237934982, + "p99": 636.9756711431546 + }, + "tpot_ms": { + "mean": 6.824520464142083, + "p50": 6.823017292777109, + "p90": 6.827681934593928, + "p95": 6.830904397375058, + "p99": 6.85009679783112 + }, + "itl_ms": { + "mean": 6.788338608930161, + "p50": 6.784363890505368, + "p90": 6.811085948498086, + "p95": 6.814371809257599, + "p99": 6.815686185589282 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_2048.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 56.44182810600614, + "request_throughput": 0.35434713352014446, + "input_token_throughput": 5805.623435594047, + "output_token_throughput": 73.65105170216202, + "total_token_throughput": 5879.2744872962085, + "total_input_tokens": 327680, + "total_output_tokens": 4157, + "e2e_ms": { + "mean": 2040.2357441023923, + "p50": 2151.03490347974, + "p90": 2357.6732086134143, + "p95": 2358.859304583166, + "p99": 2362.4721705471165 + }, + "ttft_ms": { + "mean": 630.4795558040496, + "p50": 629.4021749927197, + "p90": 636.6133440285921, + "p95": 638.6065603641327, + "p99": 639.0439760766458 + }, + "tpot_ms": { + "mean": 6.816638198034302, + "p50": 6.8195045810406665, + "p90": 6.825905331977316, + "p95": 6.848790569941936, + "p99": 6.850559613468598 + }, + "itl_ms": { + "mean": 6.788158203215443, + "p50": 6.79268100974279, + "p90": 6.8146217625671985, + "p95": 6.818863007225994, + "p99": 6.8192545745722 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 70.93397790199379, + "request_throughput": 0.2819523251273611, + "input_token_throughput": 4619.506894886685, + "output_token_throughput": 85.86858061753783, + "total_token_throughput": 4705.375475504222, + "total_input_tokens": 327680, + "total_output_tokens": 6091, + "e2e_ms": { + "mean": 2692.1835272485623, + "p50": 2038.8584179745521, + "p90": 4093.8409490103368, + "p95": 4094.3862316198647, + "p99": 4094.9791175313294 + }, + "ttft_ms": { + "mean": 621.4529997989303, + "p50": 618.6122374783736, + "p90": 628.3789116656408, + "p95": 629.9593272589846, + "p99": 645.4197830701014 + }, + "tpot_ms": { + "mean": 6.823135096274382, + "p50": 6.820567188349704, + "p90": 6.8308570155862585, + "p95": 6.8461482453307925, + "p99": 6.851399853175462 + }, + "itl_ms": { + "mean": 6.793890181817107, + "p50": 6.794746884622608, + "p90": 6.814662841437065, + "p95": 6.814756591222776, + "p99": 6.814832595126425 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_512.jsonl" + }, + { + "name": "c1_i2048_o1024", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.341512619983405, + "request_throughput": 0.46145136131632447, + "input_token_throughput": 945.0523879758325, + "output_token_throughput": 102.00382341897352, + "total_token_throughput": 1047.056211394806, + "total_input_tokens": 40960, + "total_output_tokens": 4421, + "e2e_ms": { + "mean": 1619.3287752947072, + "p50": 1505.6682314898353, + "p90": 2289.511943614343, + "p95": 2410.3983415960106, + "p99": 2903.386745112365 + }, + "ttft_ms": { + "mean": 123.14691609353758, + "p50": 121.50511099025607, + "p90": 123.98786658304743, + "p95": 127.94699885416777, + "p99": 148.69490539189425 + }, + "tpot_ms": { + "mean": 6.800605244175337, + "p50": 6.801867666012635, + "p90": 6.808000387354118, + "p95": 6.8084423111939865, + "p99": 6.809475911014002 + }, + "itl_ms": { + "mean": 6.764811909176386, + "p50": 6.765792570332284, + "p90": 6.7702946639625825, + "p95": 6.771773379877912, + "p99": 6.772758638576524 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_1024.jsonl" + }, + { + "name": "c1_i2048_o128", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 27.019237152009737, + "request_throughput": 0.7402133482703588, + "input_token_throughput": 1515.9569372576948, + "output_token_throughput": 94.11812723257611, + "total_token_throughput": 1610.075064490271, + "total_input_tokens": 40960, + "total_output_tokens": 2543, + "e2e_ms": { + "mean": 976.4232056535548, + "p50": 980.4443110188004, + "p90": 981.895874539623, + "p95": 983.6760145699373, + "p99": 1006.2214533070801 + }, + "ttft_ms": { + "mean": 122.84687320061494, + "p50": 121.22314149746671, + "p90": 123.54794240090996, + "p95": 126.64986727468205, + "p99": 147.4668486503651 + }, + "tpot_ms": { + "mean": 6.766764674291219, + "p50": 6.765895810960902, + "p90": 6.76797417273899, + "p95": 6.771030715730377, + "p99": 6.815647255658967 + }, + "itl_ms": { + "mean": 6.762888211734736, + "p50": 6.764920133919507, + "p90": 6.767030950639338, + "p95": 6.76742042006932, + "p99": 6.767525060283842 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_128.jsonl" + }, + { + "name": "c1_i2048_o2048", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.3504386190325, + "request_throughput": 0.46135634695099104, + "input_token_throughput": 944.8577985556296, + "output_token_throughput": 101.45226069452293, + "total_token_throughput": 1046.3100592501526, + "total_input_tokens": 40960, + "total_output_tokens": 4398, + "e2e_ms": { + "mean": 1611.8768365879077, + "p50": 1612.4180810002144, + "p90": 2009.0518668002917, + "p95": 2108.8781301688873, + "p99": 2496.0127236007243 + }, + "ttft_ms": { + "mean": 123.05733739340212, + "p50": 121.17295648204163, + "p90": 125.36015609512106, + "p95": 128.18555371195546, + "p99": 147.5431595003465 + }, + "tpot_ms": { + "mean": 6.80408578412217, + "p50": 6.801554977330165, + "p90": 6.812079547783474, + "p95": 6.829911392703745, + "p99": 6.835954278261701 + }, + "itl_ms": { + "mean": 6.766276811283118, + "p50": 6.767449128197479, + "p90": 6.771516465594735, + "p95": 6.771574771876405, + "p99": 6.771879487577742 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_2048.jsonl" + }, + { + "name": "c1_i2048_o256", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 37.216408888984006, + "request_throughput": 0.5373973630733611, + "input_token_throughput": 1100.5897995742434, + "output_token_throughput": 96.67778561689765, + "total_token_throughput": 1197.267585191141, + "total_input_tokens": 40960, + "total_output_tokens": 3598, + "e2e_ms": { + "mean": 1340.688644099282, + "p50": 1336.3160230219364, + "p90": 1525.1817746495365, + "p95": 1621.0493948310616, + "p99": 1775.5462661385534 + }, + "ttft_ms": { + "mean": 122.79568985104561, + "p50": 121.09551299363375, + "p90": 123.57838398893364, + "p95": 125.69494101044258, + "p99": 145.02478101814629 + }, + "tpot_ms": { + "mean": 6.808775584529611, + "p50": 6.8074119860722, + "p90": 6.816743647877187, + "p95": 6.8219026677205035, + "p99": 6.827699577873525 + }, + "itl_ms": { + "mean": 6.766320465357053, + "p50": 6.7664976447077425, + "p90": 6.769415223673198, + "p95": 6.769865434874809, + "p99": 6.770796189003204 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_256.jsonl" + }, + { + "name": "c1_i2048_o4096", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 40.114438192977104, + "request_throughput": 0.49857360344389495, + "input_token_throughput": 1021.0787398530969, + "output_token_throughput": 101.6093003818658, + "total_token_throughput": 1122.6880402349625, + "total_input_tokens": 40960, + "total_output_tokens": 4076, + "e2e_ms": { + "mean": 1502.661171098589, + "p50": 1435.3244435042143, + "p90": 2023.0349817953538, + "p95": 2076.606418809388, + "p99": 2137.6482325349934 + }, + "ttft_ms": { + "mean": 123.10377759858966, + "p50": 121.37787148822099, + "p90": 123.7979574885685, + "p95": 127.51484157342931, + "p99": 146.71904669667126 + }, + "tpot_ms": { + "mean": 6.803975411277669, + "p50": 6.804031260291214, + "p90": 6.812907689196067, + "p95": 6.816632568343603, + "p99": 6.817981062917237 + }, + "itl_ms": { + "mean": 6.765227059554421, + "p50": 6.766664099236885, + "p90": 6.770266656297683, + "p95": 6.7715020171936215, + "p99": 6.772113826350069 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_4096.jsonl" + }, + { + "name": "c1_i2048_o512", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 37.019548388023395, + "request_throughput": 0.5402551049615295, + "input_token_throughput": 1106.4424549612124, + "output_token_throughput": 96.51657450137725, + "total_token_throughput": 1202.9590294625898, + "total_input_tokens": 40960, + "total_output_tokens": 3573, + "e2e_ms": { + "mean": 1331.89303005056, + "p50": 1333.1293719820678, + "p90": 1548.5706633073278, + "p95": 1558.9390544075286, + "p99": 1578.9031204831554 + }, + "ttft_ms": { + "mean": 122.79606895172037, + "p50": 121.24446599045768, + "p90": 123.67385803372599, + "p95": 127.3277018335648, + "p99": 146.09675077954304 + }, + "tpot_ms": { + "mean": 6.806607463414278, + "p50": 6.80578571430719, + "p90": 6.8121264678222015, + "p95": 6.817936847661569, + "p99": 6.8216227906584495 + }, + "itl_ms": { + "mean": 6.76395363464037, + "p50": 6.764548409562552, + "p90": 6.76793782551663, + "p95": 6.768732991808162, + "p99": 6.769085524112825 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_512.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.121369317988865, + "request_throughput": 0.4638071637408934, + "input_token_throughput": 1899.7541426826995, + "output_token_throughput": 99.83449199522731, + "total_token_throughput": 1999.5886346779268, + "total_input_tokens": 81920, + "total_output_tokens": 4305, + "e2e_ms": { + "mean": 1644.17850044847, + "p50": 1627.3216654954012, + "p90": 2005.2758094912863, + "p95": 2109.8852323397296, + "p99": 2311.926480066613 + }, + "ttft_ms": { + "mean": 186.3455850543687, + "p50": 185.86589398910291, + "p90": 187.00031042099, + "p95": 190.6327125208918, + "p99": 190.94951929640956 + }, + "tpot_ms": { + "mean": 6.804822790357463, + "p50": 6.804042458734688, + "p90": 6.808869090739031, + "p95": 6.808880090071507, + "p99": 6.808956474964729 + }, + "itl_ms": { + "mean": 6.768538947538339, + "p50": 6.768758485865705, + "p90": 6.774978424220126, + "p95": 6.7753309364684435, + "p99": 6.777752989924467 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 28.907588530972134, + "request_throughput": 0.6918598546735099, + "input_token_throughput": 2833.8579647426964, + "output_token_throughput": 88.55806139820926, + "total_token_throughput": 2922.4160261409056, + "total_input_tokens": 81920, + "total_output_tokens": 2560, + "e2e_ms": { + "mean": 1043.4276448970195, + "p50": 1043.2338594982866, + "p90": 1044.0504656289704, + "p95": 1045.3570936457254, + "p99": 1047.5830067344941 + }, + "ttft_ms": { + "mean": 185.8626321045449, + "p50": 185.3025640011765, + "p90": 186.93417839240283, + "p95": 190.37860605167225, + "p99": 191.25218041474 + }, + "tpot_ms": { + "mean": 6.752480415688777, + "p50": 6.75444893298788, + "p90": 6.757666104827003, + "p95": 6.75770077612856, + "p99": 6.7581919096566105 + }, + "itl_ms": { + "mean": 6.751900093215627, + "p50": 6.7538843147812315, + "p90": 6.757120837780141, + "p95": 6.7571753729501465, + "p99": 6.757645178622911 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_128.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 42.9377163950121, + "request_throughput": 0.4657909567431797, + "input_token_throughput": 1907.8797588200641, + "output_token_throughput": 96.51188623718683, + "total_token_throughput": 2004.3916450572508, + "total_input_tokens": 81920, + "total_output_tokens": 4144, + "e2e_ms": { + "mean": 1589.2524215974845, + "p50": 1502.2164215042721, + "p90": 2021.3368221768178, + "p95": 2097.8192493930696, + "p99": 2452.9939467116487 + }, + "ttft_ms": { + "mean": 186.76971404929645, + "p50": 185.6516499829013, + "p90": 191.40778516884893, + "p95": 193.30401239567436, + "p99": 193.7424169119913 + }, + "tpot_ms": { + "mean": 6.802433210348238, + "p50": 6.803917665020255, + "p90": 6.808802393434643, + "p95": 6.809472329870284, + "p99": 6.809742051459609 + }, + "itl_ms": { + "mean": 6.764327205703543, + "p50": 6.764426513011319, + "p90": 6.771729031383434, + "p95": 6.772979212100163, + "p99": 6.774778441521013 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 40.42855142900953, + "request_throughput": 0.4946998913656102, + "input_token_throughput": 2026.2907550335394, + "output_token_throughput": 93.64668943551001, + "total_token_throughput": 2119.9374444690493, + "total_input_tokens": 81920, + "total_output_tokens": 3786, + "e2e_ms": { + "mean": 1469.5116658549523, + "p50": 1475.5392480292358, + "p90": 1803.493093926227, + "p95": 1913.7630071578315, + "p99": 1913.8662486354588 + }, + "ttft_ms": { + "mean": 189.04635770013556, + "p50": 186.6729814792052, + "p90": 194.76591550628658, + "p95": 207.5901315372903, + "p99": 207.82862710417248 + }, + "tpot_ms": { + "mean": 6.801977039878567, + "p50": 6.8026447482994765, + "p90": 6.814127616370421, + "p95": 6.814986886583095, + "p99": 6.821198251176533 + }, + "itl_ms": { + "mean": 6.762485412692762, + "p50": 6.76437729764892, + "p90": 6.7702427236612, + "p95": 6.772713689966694, + "p99": 6.773259632054296 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_256.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.119878391968086, + "request_throughput": 0.4638232004783526, + "input_token_throughput": 1899.8198291593324, + "output_token_throughput": 96.28969641930601, + "total_token_throughput": 1996.1095255786383, + "total_input_tokens": 81920, + "total_output_tokens": 4152, + "e2e_ms": { + "mean": 1593.6312893027207, + "p50": 1498.6803935316857, + "p90": 1957.292243535631, + "p95": 2383.6137426755163, + "p99": 2729.9412693403424 + }, + "ttft_ms": { + "mean": 187.92551439837553, + "p50": 185.15497801126912, + "p90": 193.47157318843531, + "p95": 205.29109993367456, + "p99": 207.28749755653553 + }, + "tpot_ms": { + "mean": 6.804683431925253, + "p50": 6.80432283275318, + "p90": 6.811035045306169, + "p95": 6.81644032220783, + "p99": 6.821541470131347 + }, + "itl_ms": { + "mean": 6.766404400702351, + "p50": 6.765622936847113, + "p90": 6.777374639968912, + "p95": 6.779508261833983, + "p99": 6.780120575364657 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_4096.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 41.37047825503396, + "request_throughput": 0.48343651907302765, + "input_token_throughput": 1980.1559821231212, + "output_token_throughput": 89.1940377689736, + "total_token_throughput": 2069.3500198920947, + "total_input_tokens": 81920, + "total_output_tokens": 3690, + "e2e_ms": { + "mean": 1434.7972685995046, + "p50": 1391.0726374888327, + "p90": 1746.7399196000774, + "p95": 2102.1896994323474, + "p99": 2396.88567985082 + }, + "ttft_ms": { + "mean": 185.99671100382693, + "p50": 185.7341964787338, + "p90": 186.8257091147825, + "p95": 187.35215915658046, + "p99": 190.6645454192767 + }, + "tpot_ms": { + "mean": 6.806494091020944, + "p50": 6.805695358530728, + "p90": 6.810304838032316, + "p95": 6.8114466669081, + "p99": 6.819568781625365 + }, + "itl_ms": { + "mean": 6.762844362184613, + "p50": 6.763746261668691, + "p90": 6.7704880665235025, + "p95": 6.774374872311334, + "p99": 6.777713881624482 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.877981167985126, + "request_throughput": 0.5015299023225482, + "input_token_throughput": 256.7833099891447, + "output_token_throughput": 96.79527114825181, + "total_token_throughput": 353.57858113739655, + "total_input_tokens": 10240, + "total_output_tokens": 3860, + "e2e_ms": { + "mean": 1408.4405573026743, + "p50": 1415.9690845117439, + "p90": 1817.8090272762347, + "p95": 1822.845825710101, + "p99": 1871.2951827363577 + }, + "ttft_ms": { + "mean": 118.17889174853917, + "p50": 110.91879848390818, + "p90": 127.67036562436267, + "p95": 176.59998708986677, + "p99": 184.48103661765344 + }, + "tpot_ms": { + "mean": 6.723991161659472, + "p50": 6.725678223120474, + "p90": 6.743545096530171, + "p95": 6.747081159763434, + "p99": 6.747433186365403 + }, + "itl_ms": { + "mean": 6.68911840516161, + "p50": 6.689323386256417, + "p90": 6.692883938661057, + "p95": 6.694141149762835, + "p99": 6.694941754767891 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_512_256.jsonl" + }, + { + "name": "c2_i512_o256", + "config": { + "concurrency": 2, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 41.57707812497392, + "request_throughput": 0.9620685676796844, + "input_token_throughput": 492.5791066519984, + "output_token_throughput": 196.0455223789277, + "total_token_throughput": 688.6246290309261, + "total_input_tokens": 20480, + "total_output_tokens": 8151, + "e2e_ms": { + "mean": 1706.6990505249123, + "p50": 1820.5079275066964, + "p90": 2103.1951376004145, + "p95": 2108.239625924034, + "p99": 2165.3108836821048 + }, + "ttft_ms": { + "mean": 173.5474826011341, + "p50": 182.37082049017772, + "p90": 193.2680650963448, + "p95": 211.08191983948888, + "p99": 357.8624585439684 + }, + "tpot_ms": { + "mean": 7.607090684451674, + "p50": 7.5798865189869336, + "p90": 8.037224630473764, + "p95": 8.163422641582617, + "p99": 8.396796006667133 + }, + "itl_ms": { + "mean": 7.574754799232421, + "p50": 7.555747871211839, + "p90": 7.987534884216931, + "p95": 8.11219771662063, + "p99": 8.331527962665678 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_2_512_256.jsonl" + }, + { + "name": "c32_i1024_o1024", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 101.71446370997, + "request_throughput": 6.2921238205109615, + "input_token_throughput": 6443.134792203225, + "output_token_throughput": 1317.5903909019344, + "total_token_throughput": 7760.725183105159, + "total_input_tokens": 655360, + "total_output_tokens": 134018, + "e2e_ms": { + "mean": 4796.091996566065, + "p50": 4791.799418482697, + "p90": 6029.482252488378, + "p95": 6445.2264576451835, + "p99": 7146.159083393286 + }, + "ttft_ms": { + "mean": 197.9250626429348, + "p50": 136.0024324967526, + "p90": 276.72291316557676, + "p95": 311.2744802434463, + "p99": 984.941809528973 + }, + "tpot_ms": { + "mean": 22.027192085094246, + "p50": 22.450487293471358, + "p90": 23.485227822657954, + "p95": 23.778608281911556, + "p99": 24.211396291506148 + }, + "itl_ms": { + "mean": 21.90555821158396, + "p50": 22.335648349118106, + "p90": 23.356514053533175, + "p95": 23.602322259542458, + "p99": 24.127080775726817 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_1024.jsonl" + }, + { + "name": "c32_i1024_o128", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 53.75119669100968, + "request_throughput": 11.90671165293414, + "input_token_throughput": 12192.47273260456, + "output_token_throughput": 1523.8730492059929, + "total_token_throughput": 13716.345781810553, + "total_input_tokens": 655360, + "total_output_tokens": 81910, + "e2e_ms": { + "mean": 2502.1750740123025, + "p50": 2498.9854795276187, + "p90": 2553.823436773382, + "p95": 2579.4392172567314, + "p99": 2641.3214873848483 + }, + "ttft_ms": { + "mean": 681.1004243829302, + "p50": 688.7256299960427, + "p90": 992.9460007755551, + "p95": 1046.6297078499338, + "p99": 1091.5270992362641 + }, + "tpot_ms": { + "mean": 14.341129032501167, + "p50": 14.19918624043472, + "p90": 16.83120139866268, + "p95": 16.97889331681929, + "p99": 18.599388786677757 + }, + "itl_ms": { + "mean": 14.34079579673639, + "p50": 14.198858626177682, + "p90": 16.830803370964897, + "p95": 16.97858552774781, + "p99": 18.59905340557544 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_128.jsonl" + }, + { + "name": "c32_i1024_o2048", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 101.75704183196649, + "request_throughput": 6.28949101190309, + "input_token_throughput": 6440.438796188764, + "output_token_throughput": 1316.7442526607363, + "total_token_throughput": 7757.183048849501, + "total_input_tokens": 655360, + "total_output_tokens": 133988, + "e2e_ms": { + "mean": 4767.735684630679, + "p50": 4778.584390500328, + "p90": 6232.032449706458, + "p95": 6630.186968302586, + "p99": 7109.5137719484055 + }, + "ttft_ms": { + "mean": 204.18510429781236, + "p50": 136.04777900036424, + "p90": 291.9994773750659, + "p95": 335.85453994164675, + "p99": 1145.74000693392 + }, + "tpot_ms": { + "mean": 21.89133994235568, + "p50": 22.341144466274958, + "p90": 23.580969392419064, + "p95": 24.015448060274583, + "p99": 24.526134655402384 + }, + "itl_ms": { + "mean": 21.78653640287656, + "p50": 22.26405249724315, + "p90": 23.446381741910955, + "p95": 23.931203123018353, + "p99": 24.437188434985174 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_2048.jsonl" + }, + { + "name": "c32_i1024_o256", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 99.5990810120129, + "request_throughput": 6.425762100383316, + "input_token_throughput": 6579.980390792515, + "output_token_throughput": 1309.1385851670002, + "total_token_throughput": 7889.1189759595145, + "total_input_tokens": 655360, + "total_output_tokens": 130389, + "e2e_ms": { + "mean": 4694.1599163295905, + "p50": 4782.689154468244, + "p90": 5835.8982786070555, + "p95": 6056.401502765948, + "p99": 6274.461112323916 + }, + "ttft_ms": { + "mean": 200.9333179159512, + "p50": 144.9765920115169, + "p90": 280.0194281153381, + "p95": 331.4997089968528, + "p99": 921.881116280565 + }, + "tpot_ms": { + "mean": 22.148425063318644, + "p50": 22.54947068781904, + "p90": 24.063857943282756, + "p95": 24.504361418031394, + "p99": 25.32139924061117 + }, + "itl_ms": { + "mean": 22.04392262393563, + "p50": 22.467127987028874, + "p90": 23.94794534189806, + "p95": 24.434335787560258, + "p99": 25.1582555335446 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_256.jsonl" + }, + { + "name": "c32_i1024_o4096", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 103.10473308997462, + "request_throughput": 6.207280508078153, + "input_token_throughput": 6356.2552402720285, + "output_token_throughput": 1309.6294995707576, + "total_token_throughput": 7665.884739842786, + "total_input_tokens": 655360, + "total_output_tokens": 135029, + "e2e_ms": { + "mean": 4833.693425038291, + "p50": 4832.819982984802, + "p90": 6069.454644899815, + "p95": 6458.128238443168, + "p99": 7314.038717231598 + }, + "ttft_ms": { + "mean": 205.96744189742822, + "p50": 137.8191965050064, + "p90": 290.34566325717606, + "p95": 316.60519163415296, + "p99": 1157.5180955050746 + }, + "tpot_ms": { + "mean": 22.039607651379058, + "p50": 22.46821651881679, + "p90": 23.838373629966284, + "p95": 24.052396601548065, + "p99": 24.788685409547828 + }, + "itl_ms": { + "mean": 21.923150565519745, + "p50": 22.34886148876248, + "p90": 23.690222462612827, + "p95": 23.963653610176685, + "p99": 24.723726183834795 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_4096.jsonl" + }, + { + "name": "c32_i1024_o512", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 103.0573151690187, + "request_throughput": 6.210136553143955, + "input_token_throughput": 6359.17983041941, + "output_token_throughput": 1304.3227429275164, + "total_token_throughput": 7663.5025733469265, + "total_input_tokens": 655360, + "total_output_tokens": 134420, + "e2e_ms": { + "mean": 4872.208154099917, + "p50": 4897.681755479425, + "p90": 6203.454227762995, + "p95": 6788.103030444472, + "p99": 7689.830216025585 + }, + "ttft_ms": { + "mean": 196.6105440258616, + "p50": 136.3337320217397, + "p90": 230.8978517074139, + "p95": 340.27282420720434, + "p99": 945.1565118343569 + }, + "tpot_ms": { + "mean": 22.382681351621876, + "p50": 22.651396316565084, + "p90": 24.314505681157875, + "p95": 25.381590820361843, + "p99": 27.967389366212952 + }, + "itl_ms": { + "mean": 22.267221438867587, + "p50": 22.502299660246532, + "p90": 24.191166484533962, + "p95": 25.25347770783478, + "p99": 27.81846139641072 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_512.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 398.19623553403653, + "request_throughput": 1.6072477409076231, + "input_token_throughput": 26333.146987030497, + "output_token_throughput": 402.84660096011504, + "total_token_throughput": 26735.99358799061, + "total_input_tokens": 10485760, + "total_output_tokens": 160412, + "e2e_ms": { + "mean": 19184.298590931394, + "p50": 16635.307037999155, + "p90": 30508.484355715336, + "p95": 36508.48627480736, + "p99": 54198.776468519965 + }, + "ttft_ms": { + "mean": 1772.2575316559414, + "p50": 1323.4945705044083, + "p90": 2576.021732011577, + "p95": 3387.5279530038806, + "p99": 13518.463227089967 + }, + "tpot_ms": { + "mean": 70.46074460640907, + "p50": 71.73240135433585, + "p90": 82.03219558137306, + "p95": 85.49271957200986, + "p99": 101.8386985072882 + }, + "itl_ms": { + "mean": 70.11671918436993, + "p50": 71.3985684091518, + "p90": 81.70230217048596, + "p95": 85.39689406407119, + "p99": 100.46498175445457 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 359.2006303339731, + "request_throughput": 1.781734067128303, + "input_token_throughput": 29191.930955830117, + "output_token_throughput": 227.5497120481234, + "total_token_throughput": 29419.48066787824, + "total_input_tokens": 10485760, + "total_output_tokens": 81736, + "e2e_ms": { + "mean": 17518.50459199386, + "p50": 17511.15690698498, + "p90": 18241.288167028688, + "p95": 19731.453384371707, + "p99": 27923.40092844214 + }, + "ttft_ms": { + "mean": 3438.601437185207, + "p50": 3365.2316914813127, + "p90": 3878.172927122796, + "p95": 3983.9240162749716, + "p99": 13516.256364046138 + }, + "tpot_ms": { + "mean": 111.1261431259428, + "p50": 111.30386603538747, + "p90": 126.74013408946834, + "p95": 127.94849833767834, + "p99": 130.66239218253233 + }, + "itl_ms": { + "mean": 111.11157190744719, + "p50": 111.30349909062645, + "p90": 126.73962503148236, + "p95": 127.94812049980955, + "p99": 130.66199353753544 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_128.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 398.6252235029824, + "request_throughput": 1.605518071274814, + "input_token_throughput": 26304.808079766553, + "output_token_throughput": 403.5168637510881, + "total_token_throughput": 26708.32494351764, + "total_input_tokens": 10485760, + "total_output_tokens": 160852, + "e2e_ms": { + "mean": 19175.977931623493, + "p50": 16826.667610002914, + "p90": 31094.610782997916, + "p95": 37536.35124927386, + "p99": 50504.81318892155 + }, + "ttft_ms": { + "mean": 1715.0379632086697, + "p50": 1321.1069589888211, + "p90": 2372.458817012375, + "p95": 2963.249287707724, + "p99": 13514.417622748886 + }, + "tpot_ms": { + "mean": 70.47087580375502, + "p50": 71.53828355218127, + "p90": 81.78005991342835, + "p95": 86.6475722402378, + "p99": 95.97500238836932 + }, + "itl_ms": { + "mean": 70.14611375684596, + "p50": 71.22865627701796, + "p90": 81.27930098275836, + "p95": 86.08798771959307, + "p99": 95.6376215260082 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 383.92433379095746, + "request_throughput": 1.6669951437578658, + "input_token_throughput": 27312.048435328874, + "output_token_throughput": 348.16235449347874, + "total_token_throughput": 27660.21078982235, + "total_input_tokens": 10485760, + "total_output_tokens": 133668, + "e2e_ms": { + "mean": 18646.62216086317, + "p50": 18877.446957485517, + "p90": 23596.388958598254, + "p95": 24405.538778216567, + "p99": 29695.085078190335 + }, + "ttft_ms": { + "mean": 1826.8149450027522, + "p50": 1393.023353011813, + "p90": 2421.4535025763325, + "p95": 3388.6907908861745, + "p99": 13517.466220164213 + }, + "tpot_ms": { + "mean": 80.78933965516552, + "p50": 82.40988242028433, + "p90": 90.68880433252747, + "p95": 92.926508280831, + "p99": 101.6337257692619 + }, + "itl_ms": { + "mean": 80.45402753619312, + "p50": 82.00862929269339, + "p90": 90.5916024319385, + "p95": 92.75806571797227, + "p99": 101.5516533698931 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_256.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 393.371598202968, + "request_throughput": 1.6269603675600877, + "input_token_throughput": 26656.118662104476, + "output_token_throughput": 383.78215582840454, + "total_token_throughput": 27039.90081793288, + "total_input_tokens": 10485760, + "total_output_tokens": 150969, + "e2e_ms": { + "mean": 19000.893150751017, + "p50": 16956.22364000883, + "p90": 30767.245342204118, + "p95": 36338.548749694, + "p99": 41160.94916441828 + }, + "ttft_ms": { + "mean": 1747.0441726392892, + "p50": 1320.0604559970088, + "p90": 2374.9253470858093, + "p95": 3400.819807735387, + "p99": 13513.480063019677 + }, + "tpot_ms": { + "mean": 73.78208413979608, + "p50": 74.50801520232787, + "p90": 85.94058834740318, + "p95": 90.2959977150635, + "p99": 96.95683291620503 + }, + "itl_ms": { + "mean": 73.42788346346373, + "p50": 74.34369396839529, + "p90": 85.69958084748383, + "p95": 89.27350224578593, + "p99": 96.65062597269109 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_512.jsonl" + }, + { + "name": "c32_i2048_o1024", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 112.12105303100543, + "request_throughput": 5.708116207426427, + "input_token_throughput": 11690.221992809322, + "output_token_throughput": 1270.4482891416405, + "total_token_throughput": 12960.670281950963, + "total_input_tokens": 1310720, + "total_output_tokens": 142444, + "e2e_ms": { + "mean": 5165.109592623776, + "p50": 4740.339960466372, + "p90": 7289.129797718488, + "p95": 8407.833625149215, + "p99": 11262.115388974782 + }, + "ttft_ms": { + "mean": 246.5300903953903, + "p50": 154.3558800185565, + "p90": 310.4189208941534, + "p95": 373.10424581228256, + "p99": 1896.2616414064541 + }, + "tpot_ms": { + "mean": 22.207007231194755, + "p50": 22.51868323658376, + "p90": 23.79985572262988, + "p95": 24.254527229962846, + "p99": 25.251460401786787 + }, + "itl_ms": { + "mean": 22.094839150458522, + "p50": 22.41546275374803, + "p90": 23.680054680674566, + "p95": 24.062508614105823, + "p99": 25.073584625372547 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_1024.jsonl" + }, + { + "name": "c32_i2048_o128", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 73.91267868096475, + "request_throughput": 8.658866265184132, + "input_token_throughput": 17733.358111097103, + "output_token_throughput": 1105.4125145790692, + "total_token_throughput": 18838.77062567617, + "total_input_tokens": 1310720, + "total_output_tokens": 81704, + "e2e_ms": { + "mean": 3492.798290261726, + "p50": 3497.8387630253565, + "p90": 3721.036367805209, + "p95": 3746.523601966328, + "p99": 4348.107912972922 + }, + "ttft_ms": { + "mean": 904.6822240371512, + "p50": 975.327655498404, + "p90": 1452.6418834982906, + "p95": 1464.6129804546945, + "p99": 1844.7583691641953 + }, + "tpot_ms": { + "mean": 20.425485261385226, + "p50": 19.787422429007574, + "p90": 25.10513298210732, + "p95": 26.381000489116587, + "p99": 27.057591470652333 + }, + "itl_ms": { + "mean": 20.42322585400549, + "p50": 19.787048114102525, + "p90": 25.104814078878817, + "p95": 26.38060599730504, + "p99": 27.057244394508036 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_128.jsonl" + }, + { + "name": "c32_i2048_o2048", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 111.78773300204193, + "request_throughput": 5.725136227499217, + "input_token_throughput": 11725.078993918396, + "output_token_throughput": 1282.421569434469, + "total_token_throughput": 13007.500563352864, + "total_input_tokens": 1310720, + "total_output_tokens": 143359, + "e2e_ms": { + "mean": 5185.733147027804, + "p50": 4744.378389994381, + "p90": 7545.672788115918, + "p95": 8810.36796895496, + "p99": 10858.06654340995 + }, + "ttft_ms": { + "mean": 246.50183836502038, + "p50": 150.90322599280626, + "p90": 331.4385375881102, + "p95": 442.18197880545597, + "p99": 1847.0357541838925 + }, + "tpot_ms": { + "mean": 22.172976606138974, + "p50": 22.429171546363605, + "p90": 23.695601993977686, + "p95": 24.280749206855816, + "p99": 25.62301267945426 + }, + "itl_ms": { + "mean": 22.066964132193903, + "p50": 22.299661700202833, + "p90": 23.59680629773218, + "p95": 24.194506006747183, + "p99": 25.525007229595307 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_2048.jsonl" + }, + { + "name": "c32_i2048_o256", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 103.61332845099969, + "request_throughput": 6.176811512262785, + "input_token_throughput": 12650.109977114183, + "output_token_throughput": 1258.1103411000622, + "total_token_throughput": 13908.220318214246, + "total_input_tokens": 1310720, + "total_output_tokens": 130357, + "e2e_ms": { + "mean": 4859.842472118635, + "p50": 4828.371057519689, + "p90": 6167.06687019323, + "p95": 6298.454219402629, + "p99": 6687.988225959125 + }, + "ttft_ms": { + "mean": 240.9834422237509, + "p50": 186.60073648788966, + "p90": 304.38226247788407, + "p95": 367.88477115624113, + "p99": 1848.7024522264287 + }, + "tpot_ms": { + "mean": 22.795846379865804, + "p50": 23.15601111415637, + "p90": 24.32344850623629, + "p95": 24.73872366202562, + "p99": 25.469112502702256 + }, + "itl_ms": { + "mean": 22.696593278349937, + "p50": 23.06816904172357, + "p90": 24.253067279885506, + "p95": 24.58853132190758, + "p99": 25.373207823111766 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_256.jsonl" + }, + { + "name": "c32_i2048_o4096", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 106.83542463398771, + "request_throughput": 5.990522358970396, + "input_token_throughput": 12268.589791171371, + "output_token_throughput": 1282.1496284521966, + "total_token_throughput": 13550.739419623567, + "total_input_tokens": 1310720, + "total_output_tokens": 136979, + "e2e_ms": { + "mean": 5036.072500929913, + "p50": 4737.078030011617, + "p90": 6963.614087650785, + "p95": 8060.132782527941, + "p99": 10159.852220952163 + }, + "ttft_ms": { + "mean": 250.26007797550847, + "p50": 189.39865499851294, + "p90": 331.10389779903926, + "p95": 439.4650830334279, + "p99": 1837.7938479348118 + }, + "tpot_ms": { + "mean": 22.45339401983756, + "p50": 22.80440429761633, + "p90": 24.29076330914555, + "p95": 24.64607409221561, + "p99": 25.843275491954916 + }, + "itl_ms": { + "mean": 22.335401602134166, + "p50": 22.700434707873512, + "p90": 24.13671753592239, + "p95": 24.522132348434965, + "p99": 25.597794042088253 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_4096.jsonl" + }, + { + "name": "c32_i2048_o512", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 111.05525721795857, + "request_throughput": 5.7628969220604045, + "input_token_throughput": 11802.412896379708, + "output_token_throughput": 1281.6502664133527, + "total_token_throughput": 13084.063162793062, + "total_input_tokens": 1310720, + "total_output_tokens": 142334, + "e2e_ms": { + "mean": 5136.635271709019, + "p50": 4666.2061315146275, + "p90": 7534.419533464826, + "p95": 8816.123298896124, + "p99": 11310.105777755381 + }, + "ttft_ms": { + "mean": 242.25743507849984, + "p50": 148.94087848369963, + "p90": 326.93816607352346, + "p95": 437.2403310961068, + "p99": 1893.1734945153585 + }, + "tpot_ms": { + "mean": 22.15493830038853, + "p50": 22.43389434124179, + "p90": 23.98985761731914, + "p95": 24.51438626808776, + "p99": 25.546152305927095 + }, + "itl_ms": { + "mean": 22.048586154855595, + "p50": 22.311854000107793, + "p90": 23.930691744822965, + "p95": 24.459427310307202, + "p99": 25.435365217090617 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_512.jsonl" + }, + { + "name": "c32_i4096_o1024", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 141.80376432603225, + "request_throughput": 4.513279340938547, + "input_token_throughput": 18486.39218048429, + "output_token_throughput": 947.3161776660905, + "total_token_throughput": 19433.708358150383, + "total_input_tokens": 2621440, + "total_output_tokens": 134333, + "e2e_ms": { + "mean": 6702.716318087732, + "p50": 6253.930084494641, + "p90": 9191.221753915306, + "p95": 11605.60482868459, + "p99": 14590.752592555365 + }, + "ttft_ms": { + "mean": 432.84981709230124, + "p50": 304.1287414962426, + "p90": 594.3656270916108, + "p95": 813.4859444078755, + "p99": 3483.0619139934424 + }, + "tpot_ms": { + "mean": 30.04166296734717, + "p50": 30.51881899689274, + "p90": 33.11392199115068, + "p95": 33.92001727738661, + "p99": 36.32087159977741 + }, + "itl_ms": { + "mean": 29.864745135826638, + "p50": 30.36130819243728, + "p90": 32.89914557815899, + "p95": 33.71280012456758, + "p99": 36.25344940904338 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_1024.jsonl" + }, + { + "name": "c32_i4096_o128", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 114.34404653601814, + "request_throughput": 5.597143177877664, + "input_token_throughput": 22925.89845658691, + "output_token_throughput": 710.9771121699102, + "total_token_throughput": 23636.87556875682, + "total_input_tokens": 2621440, + "total_output_tokens": 81296, + "e2e_ms": { + "mean": 5478.886110145686, + "p50": 5515.89411549503, + "p90": 5904.106940201018, + "p95": 6106.497004799892, + "p99": 7458.033464258187 + }, + "ttft_ms": { + "mean": 1053.8112139072837, + "p50": 996.5255100105423, + "p90": 1746.4399764954578, + "p95": 1909.4978857465321, + "p99": 3485.2048156253295 + }, + "tpot_ms": { + "mean": 35.09868246136891, + "p50": 35.47592700402231, + "p90": 41.460134331586104, + "p95": 41.98311302248539, + "p99": 43.261149156257495 + }, + "itl_ms": { + "mean": 35.06339184305249, + "p50": 35.37396917313953, + "p90": 41.45976171093648, + "p95": 41.9827239108721, + "p99": 43.204316071992984 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_128.jsonl" + }, + { + "name": "c32_i4096_o2048", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 144.45277938101208, + "request_throughput": 4.430513575041162, + "input_token_throughput": 18147.3836033686, + "output_token_throughput": 939.6703932014643, + "total_token_throughput": 19087.053996570063, + "total_input_tokens": 2621440, + "total_output_tokens": 135738, + "e2e_ms": { + "mean": 6700.040520485527, + "p50": 6357.520977006061, + "p90": 9260.33614158514, + "p95": 10714.146034815345, + "p99": 14733.258541694968 + }, + "ttft_ms": { + "mean": 434.39145126149015, + "p50": 309.5579859800637, + "p90": 571.2298724334687, + "p95": 747.313593828585, + "p99": 3483.196152728633 + }, + "tpot_ms": { + "mean": 29.876116420692682, + "p50": 30.3379228851805, + "p90": 32.78072649504329, + "p95": 33.53012496942364, + "p99": 35.93158563458935 + }, + "itl_ms": { + "mean": 29.728421008128368, + "p50": 30.180660484182958, + "p90": 32.61751984353158, + "p95": 33.27038472473239, + "p99": 35.61634739842644 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_2048.jsonl" + }, + { + "name": "c32_i4096_o256", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 137.277509423031, + "request_throughput": 4.662089243095107, + "input_token_throughput": 19095.917539717557, + "output_token_throughput": 921.5056459844019, + "total_token_throughput": 20017.42318570196, + "total_input_tokens": 2621440, + "total_output_tokens": 126502, + "e2e_ms": { + "mean": 6528.498834509901, + "p50": 6445.90070299455, + "p90": 8382.710035092896, + "p95": 8638.599968180642, + "p99": 10034.495897529998 + }, + "ttft_ms": { + "mean": 440.9964826984833, + "p50": 307.16633147676475, + "p90": 597.6938761305064, + "p95": 731.7515331407769, + "p99": 3478.6799135484034 + }, + "tpot_ms": { + "mean": 30.982427560307695, + "p50": 31.59422003819917, + "p90": 33.60870284218423, + "p95": 34.415974134281406, + "p99": 36.57922350277463 + }, + "itl_ms": { + "mean": 30.833046346702986, + "p50": 31.449339798004964, + "p90": 33.47908859361284, + "p95": 34.3315886835987, + "p99": 36.18148070887706 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_256.jsonl" + }, + { + "name": "c32_i4096_o4096", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 143.00942956196377, + "request_throughput": 4.475229374456724, + "input_token_throughput": 18330.53951777474, + "output_token_throughput": 961.5030311020271, + "total_token_throughput": 19292.04254887677, + "total_input_tokens": 2621440, + "total_output_tokens": 137504, + "e2e_ms": { + "mean": 6743.487641989396, + "p50": 6318.3071019884665, + "p90": 9436.950038914803, + "p95": 10500.857756665206, + "p99": 14281.367835595158 + }, + "ttft_ms": { + "mean": 429.60839072684394, + "p50": 308.00414050463587, + "p90": 571.643524611136, + "p95": 749.1604033275432, + "p99": 3469.335957507137 + }, + "tpot_ms": { + "mean": 29.60218774118301, + "p50": 30.001832055703364, + "p90": 32.42566294262988, + "p95": 33.17141477764495, + "p99": 35.01187529090304 + }, + "itl_ms": { + "mean": 29.463924546216248, + "p50": 29.88822948908127, + "p90": 32.286102683108695, + "p95": 33.004748232670295, + "p99": 34.77493098066052 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_4096.jsonl" + }, + { + "name": "c32_i4096_o512", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 143.19192939798813, + "request_throughput": 4.46952564080048, + "input_token_throughput": 18307.177024718767, + "output_token_throughput": 929.291200694371, + "total_token_throughput": 19236.468225413137, + "total_input_tokens": 2621440, + "total_output_tokens": 133067, + "e2e_ms": { + "mean": 6735.349410550407, + "p50": 6316.515762504423, + "p90": 9159.898966323817, + "p95": 10401.616307868968, + "p99": 14892.06140367722 + }, + "ttft_ms": { + "mean": 430.74002514895255, + "p50": 310.50263551878743, + "p90": 543.8166258158163, + "p95": 691.6757039667568, + "p99": 3532.9511326598004 + }, + "tpot_ms": { + "mean": 30.56646949473112, + "p50": 30.9312377398962, + "p90": 33.449419883316686, + "p95": 34.146055170403926, + "p99": 36.9598451346281 + }, + "itl_ms": { + "mean": 30.39183315845209, + "p50": 30.743593022214746, + "p90": 33.260713398804185, + "p95": 33.86867077639849, + "p99": 36.898599968271185 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 98.72821586497594, + "request_throughput": 6.482442677534918, + "input_token_throughput": 3319.010650897878, + "output_token_throughput": 1330.8150952478677, + "total_token_throughput": 4649.825746145746, + "total_input_tokens": 327680, + "total_output_tokens": 131389, + "e2e_ms": { + "mean": 4649.840651139948, + "p50": 4760.1729684975, + "p90": 5945.521820668364, + "p95": 6039.506064294255, + "p99": 6163.958358393284 + }, + "ttft_ms": { + "mean": 186.9829640457283, + "p50": 136.8874144682195, + "p90": 252.83180618425908, + "p95": 378.34352593054064, + "p99": 678.2834628055571 + }, + "tpot_ms": { + "mean": 21.818303136745314, + "p50": 22.270404707809327, + "p90": 23.48358913604192, + "p95": 23.81546439573604, + "p99": 24.53757630767593 + }, + "itl_ms": { + "mean": 21.733008667438355, + "p50": 22.199319917640157, + "p90": 23.35130398678313, + "p95": 23.740902679644243, + "p99": 24.421781921266398 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_512_256.jsonl" + }, + { + "name": "c4_i512_o256", + "config": { + "concurrency": 4, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "metrics": { + "success": 80, + "failed": 0, + "duration_s": 46.0010106950067, + "request_throughput": 1.7390922240907156, + "input_token_throughput": 890.4152187344464, + "output_token_throughput": 360.4703407484031, + "total_token_throughput": 1250.8855594828497, + "total_input_tokens": 40960, + "total_output_tokens": 16582, + "e2e_ms": { + "mean": 1997.4373223354633, + "p50": 2098.9274029852822, + "p90": 2478.9669486810453, + "p95": 2523.6370165017433, + "p99": 2675.4348404653138 + }, + "ttft_ms": { + "mean": 164.63600002534804, + "p50": 118.1012575107161, + "p90": 209.9170045054044, + "p95": 347.47324945928995, + "p99": 374.55094301200006 + }, + "tpot_ms": { + "mean": 8.916417548236618, + "p50": 8.663080307750908, + "p90": 10.1927724996491, + "p95": 10.51017343682637, + "p99": 11.091826626165526 + }, + "itl_ms": { + "mean": 8.863431249984234, + "p50": 8.662887984275013, + "p90": 10.07585571055859, + "p95": 10.268174330863944, + "p99": 10.95105848389873 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_4_512_256.jsonl" + }, + { + "name": "c64_i512_o256", + "config": { + "concurrency": 64, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 1280 + }, + "metrics": { + "success": 1280, + "failed": 0, + "duration_s": 140.30180359998485, + "request_throughput": 9.12318991742554, + "input_token_throughput": 4671.0732377218765, + "output_token_throughput": 1901.52936850791, + "total_token_throughput": 6572.602606229786, + "total_input_tokens": 655360, + "total_output_tokens": 266788, + "e2e_ms": { + "mean": 6712.706156762669, + "p50": 6909.8945554869715, + "p90": 8549.926256021718, + "p95": 8741.368123874418, + "p99": 9237.345241245348 + }, + "ttft_ms": { + "mean": 229.05817989149, + "p50": 194.47254750411958, + "p90": 306.9184340885839, + "p95": 453.78009574778844, + "p99": 1147.805790141574 + }, + "tpot_ms": { + "mean": 31.257536585577196, + "p50": 32.00854534300386, + "p90": 34.015025776029766, + "p95": 35.352318105270726, + "p99": 36.91540531818083 + }, + "itl_ms": { + "mean": 31.135016112704886, + "p50": 31.86686346661208, + "p90": 33.88631216092506, + "p95": 35.253558867598436, + "p99": 36.734858386072254 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_64_512_256.jsonl" + }, + { + "name": "c8_i1024_o1024", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 54.01378972904058, + "request_throughput": 2.962206518050997, + "input_token_throughput": 3033.299474484221, + "output_token_throughput": 621.7671481389042, + "total_token_throughput": 3655.066622623125, + "total_input_tokens": 163840, + "total_output_tokens": 33584, + "e2e_ms": { + "mean": 2395.4786735364905, + "p50": 2329.696138011059, + "p90": 2964.6093561896123, + "p95": 3214.61368437449, + "p99": 3506.79572086723 + }, + "ttft_ms": { + "mean": 144.65098456239502, + "p50": 123.20308148628101, + "p90": 199.4970897270832, + "p95": 217.99946820829152, + "p99": 393.9990232139826 + }, + "tpot_ms": { + "mean": 10.80428496965314, + "p50": 10.846479013653251, + "p90": 11.449212558667767, + "p95": 11.820453448058938, + "p99": 12.236586239159527 + }, + "itl_ms": { + "mean": 10.748665729836475, + "p50": 10.797932864638046, + "p90": 11.402054409661389, + "p95": 11.773988858873494, + "p99": 12.175650934477838 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_1024.jsonl" + }, + { + "name": "c8_i1024_o128", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 30.886076872993726, + "request_throughput": 5.180327713938358, + "input_token_throughput": 5304.655579072879, + "output_token_throughput": 662.822930998413, + "total_token_throughput": 5967.4785100712925, + "total_input_tokens": 163840, + "total_output_tokens": 20472, + "e2e_ms": { + "mean": 1360.7545400929666, + "p50": 1356.6849144699518, + "p90": 1372.686949709896, + "p95": 1376.9633801421153, + "p99": 1412.428418964264 + }, + "ttft_ms": { + "mean": 307.6379093559808, + "p50": 328.5264960140921, + "p90": 341.3232935068663, + "p95": 345.5637474049581, + "p99": 389.5727182831615 + }, + "tpot_ms": { + "mean": 8.295535728493961, + "p50": 8.095632850579316, + "p90": 9.738737157010656, + "p95": 9.771883254412444, + "p99": 9.797184904290209 + }, + "itl_ms": { + "mean": 8.295176800331998, + "p50": 8.095306578765102, + "p90": 9.73838070324094, + "p95": 9.771548070340769, + "p99": 9.796842210201957 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_128.jsonl" + }, + { + "name": "c8_i1024_o2048", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 53.44451181899058, + "request_throughput": 2.993759219691231, + "input_token_throughput": 3065.6094409638204, + "output_token_throughput": 627.0802905545745, + "total_token_throughput": 3692.689731518395, + "total_input_tokens": 163840, + "total_output_tokens": 33514, + "e2e_ms": { + "mean": 2391.8724685270718, + "p50": 2429.516384989256, + "p90": 2975.693142216187, + "p95": 3190.5777254287395, + "p99": 3558.5536541033066 + }, + "ttft_ms": { + "mean": 147.13870290943305, + "p50": 123.122916003922, + "p90": 209.92188203381374, + "p95": 284.30513975326926, + "p99": 375.40317902341485 + }, + "tpot_ms": { + "mean": 10.762963745519746, + "p50": 10.834037995907572, + "p90": 11.480905401984911, + "p95": 11.595140581413345, + "p99": 12.305920331146154 + }, + "itl_ms": { + "mean": 10.704678773876426, + "p50": 10.78597629218042, + "p90": 11.407551552441637, + "p95": 11.554190884636792, + "p99": 12.25965226868399 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_2048.jsonl" + }, + { + "name": "c8_i1024_o256", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 52.3178577030194, + "request_throughput": 3.0582291979200438, + "input_token_throughput": 3131.626698670125, + "output_token_throughput": 620.5529321229509, + "total_token_throughput": 3752.179630793076, + "total_input_tokens": 163840, + "total_output_tokens": 32466, + "e2e_ms": { + "mean": 2345.425173519834, + "p50": 2381.8502814974636, + "p90": 2948.787523934152, + "p95": 2965.105928858975, + "p99": 3042.159925229498 + }, + "ttft_ms": { + "mean": 144.05868405883666, + "p50": 123.30061200191267, + "p90": 205.80871163983826, + "p95": 216.85114067804534, + "p99": 383.2814864401007 + }, + "tpot_ms": { + "mean": 10.888348926023786, + "p50": 10.947267107515419, + "p90": 11.52796358664845, + "p95": 11.752466147783078, + "p99": 12.285815981373782 + }, + "itl_ms": { + "mean": 10.835623968999094, + "p50": 10.886456865874756, + "p90": 11.473180943125312, + "p95": 11.660246153027902, + "p99": 12.233038689088968 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_256.jsonl" + }, + { + "name": "c8_i1024_o4096", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 55.321856701979414, + "request_throughput": 2.8921661263454164, + "input_token_throughput": 2961.5781133777064, + "output_token_throughput": 621.4903484750506, + "total_token_throughput": 3583.0684618527566, + "total_input_tokens": 163840, + "total_output_tokens": 34382, + "e2e_ms": { + "mean": 2438.4150670175586, + "p50": 2406.0954789747484, + "p90": 3173.2560309872497, + "p95": 3382.0623315201383, + "p99": 3756.3808398373653 + }, + "ttft_ms": { + "mean": 144.43321304788697, + "p50": 123.22676149779, + "p90": 207.3197172721848, + "p95": 220.03079180431183, + "p99": 380.43375951296184 + }, + "tpot_ms": { + "mean": 10.729683978884637, + "p50": 10.795890332354045, + "p90": 11.431676349473312, + "p95": 11.55948920758737, + "p99": 12.092195742100818 + }, + "itl_ms": { + "mean": 10.677337930646178, + "p50": 10.729629197808688, + "p90": 11.383656778432892, + "p95": 11.513900642614868, + "p99": 12.038068449618027 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_4096.jsonl" + }, + { + "name": "c8_i1024_o512", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 54.03436242096359, + "request_throughput": 2.961078706795755, + "input_token_throughput": 3032.144595758853, + "output_token_throughput": 632.2088106428112, + "total_token_throughput": 3664.353406401664, + "total_input_tokens": 163840, + "total_output_tokens": 34161, + "e2e_ms": { + "mean": 2425.822611462354, + "p50": 2396.973063965561, + "p90": 3130.640160792973, + "p95": 3208.353921084199, + "p99": 3616.7711639672048 + }, + "ttft_ms": { + "mean": 144.40323555427312, + "p50": 123.08634052169509, + "p90": 203.37686730199493, + "p95": 216.0994939156808, + "p99": 375.8623972674832 + }, + "tpot_ms": { + "mean": 10.739382575641525, + "p50": 10.858331415132689, + "p90": 11.355513657332015, + "p95": 11.482381241675077, + "p99": 11.759983664804802 + }, + "itl_ms": { + "mean": 10.685416116247126, + "p50": 10.819440029603811, + "p90": 11.31777107405648, + "p95": 11.439431671480277, + "p99": 11.715367241071098 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_512.jsonl" + }, + { + "name": "c8_i16384_o1024", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 138.9691290059709, + "request_throughput": 1.151334840654614, + "input_token_throughput": 18863.470029285196, + "output_token_throughput": 287.5890515100144, + "total_token_throughput": 19151.05908079521, + "total_input_tokens": 2621440, + "total_output_tokens": 39966, + "e2e_ms": { + "mean": 6298.755326521496, + "p50": 5593.749911990017, + "p90": 10127.106156485386, + "p95": 12516.83689956845, + "p99": 14588.597605068933 + }, + "ttft_ms": { + "mean": 967.7164456628816, + "p50": 890.4675914964173, + "p90": 1378.9012019173242, + "p95": 1600.4013906058397, + "p99": 3663.330937565768 + }, + "tpot_ms": { + "mean": 21.472376172379033, + "p50": 22.257800259102922, + "p90": 26.010914360549727, + "p95": 27.316078669897212, + "p99": 29.59252122410235 + }, + "itl_ms": { + "mean": 21.351223905498287, + "p50": 22.178765532877954, + "p90": 25.717352797940066, + "p95": 27.31455100890858, + "p99": 29.54132950093783 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_1024.jsonl" + }, + { + "name": "c8_i16384_o128", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 111.92959954298567, + "request_throughput": 1.4294699583781971, + "input_token_throughput": 23420.43579806838, + "output_token_throughput": 181.32826422027432, + "total_token_throughput": 23601.764062288657, + "total_input_tokens": 2621440, + "total_output_tokens": 20296, + "e2e_ms": { + "mean": 5167.535090905221, + "p50": 5161.67644050438, + "p90": 5232.32137238374, + "p95": 5417.745105508948, + "p99": 5933.665734616807 + }, + "ttft_ms": { + "mean": 2556.643709580385, + "p50": 2640.1203620189335, + "p90": 3866.4832159643993, + "p95": 4119.31119611545, + "p99": 4136.104622313287 + }, + "tpot_ms": { + "mean": 20.737635027964952, + "p50": 20.266374896487427, + "p90": 33.92593250609934, + "p95": 33.96777861103941, + "p99": 33.99695902494602 + }, + "itl_ms": { + "mean": 20.736728008977952, + "p50": 20.266098003799378, + "p90": 33.925570322973066, + "p95": 33.967433898780854, + "p99": 33.99658160026847 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_128.jsonl" + }, + { + "name": "c8_i16384_o2048", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 136.25688857299974, + "request_throughput": 1.1742525583525258, + "input_token_throughput": 19238.953916047783, + "output_token_throughput": 275.00994916616156, + "total_token_throughput": 19513.963865213947, + "total_input_tokens": 2621440, + "total_output_tokens": 37472, + "e2e_ms": { + "mean": 6163.9701214990055, + "p50": 5399.014000024181, + "p90": 9542.940336407628, + "p95": 11684.397922272918, + "p99": 14272.990592987044 + }, + "ttft_ms": { + "mean": 961.941650908193, + "p50": 889.2011054849718, + "p90": 1358.9895943936426, + "p95": 1572.8736859047788, + "p99": 3673.563928353831 + }, + "tpot_ms": { + "mean": 22.523256550346055, + "p50": 22.868919465467282, + "p90": 26.401673455046737, + "p95": 28.431650291592916, + "p99": 30.67564380545814 + }, + "itl_ms": { + "mean": 22.396629750278695, + "p50": 22.68798743846689, + "p90": 26.286321661482393, + "p95": 27.869237309002852, + "p99": 30.610476568286856 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_2048.jsonl" + }, + { + "name": "c8_i16384_o256", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 127.81865492399083, + "request_throughput": 1.2517734605730773, + "input_token_throughput": 20509.056378029298, + "output_token_throughput": 253.68753895339125, + "total_token_throughput": 20762.743916982687, + "total_input_tokens": 2621440, + "total_output_tokens": 32426, + "e2e_ms": { + "mean": 5890.637765062638, + "p50": 5999.108154996065, + "p90": 7295.3310133947525, + "p95": 7584.774667481543, + "p99": 7970.696000051683 + }, + "ttft_ms": { + "mean": 1027.092823043131, + "p50": 896.6540365072433, + "p90": 1433.758345700334, + "p95": 1938.521144268454, + "p99": 3668.7240775686205 + }, + "tpot_ms": { + "mean": 24.18784923644043, + "p50": 24.955616296678972, + "p90": 28.24579013451567, + "p95": 29.34601891272757, + "p99": 30.471414801470594 + }, + "itl_ms": { + "mean": 24.09058552564381, + "p50": 24.940828342397996, + "p90": 28.163052283118482, + "p95": 29.2733077929844, + "p99": 30.42853842177255 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_256.jsonl" + }, + { + "name": "c8_i16384_o512", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 136.6128168310388, + "request_throughput": 1.1711931845888677, + "input_token_throughput": 19188.829136304008, + "output_token_throughput": 286.08589520804196, + "total_token_throughput": 19474.91503151205, + "total_input_tokens": 2621440, + "total_output_tokens": 39083, + "e2e_ms": { + "mean": 6250.800366418844, + "p50": 5613.646615500329, + "p90": 9697.346573503455, + "p95": 10966.727837751389, + "p99": 12572.724322149297 + }, + "ttft_ms": { + "mean": 969.4523646234302, + "p50": 872.605326003395, + "p90": 1416.2633731088135, + "p95": 1904.348115285393, + "p99": 3667.981622895457 + }, + "tpot_ms": { + "mean": 21.65413662900705, + "p50": 22.15968788136079, + "p90": 25.971828724872314, + "p95": 26.893490572898347, + "p99": 29.249297661201627 + }, + "itl_ms": { + "mean": 21.52535903197245, + "p50": 22.04750119635838, + "p90": 25.842861547491193, + "p95": 26.289499275910167, + "p99": 29.012356170540826 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_512.jsonl" + }, + { + "name": "c8_i2048_o1024", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 59.21719591296278, + "request_throughput": 2.7019178725579547, + "input_token_throughput": 5533.527802998691, + "output_token_throughput": 606.8169802031097, + "total_token_throughput": 6140.344783201801, + "total_input_tokens": 327680, + "total_output_tokens": 35934, + "e2e_ms": { + "mean": 2572.1823010015214, + "p50": 2315.6313379877247, + "p90": 3573.3853886660645, + "p95": 4486.641788340058, + "p99": 5823.30393439799 + }, + "ttft_ms": { + "mean": 155.89313780001248, + "p50": 127.48752950574271, + "p90": 214.49003661982712, + "p95": 244.40497531904856, + "p99": 621.821830929839 + }, + "tpot_ms": { + "mean": 10.829963419578188, + "p50": 10.84090427110567, + "p90": 11.683323234763234, + "p95": 11.858814439893099, + "p99": 12.107412703150931 + }, + "itl_ms": { + "mean": 10.77232341137624, + "p50": 10.792881839767215, + "p90": 11.5925756148027, + "p95": 11.796772338880846, + "p99": 12.05002575920413 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_1024.jsonl" + }, + { + "name": "c8_i2048_o128", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 36.785884183016606, + "request_throughput": 4.349494474673228, + "input_token_throughput": 8907.76468413077, + "output_token_throughput": 554.3702551375695, + "total_token_throughput": 9462.134939268339, + "total_input_tokens": 327680, + "total_output_tokens": 20393, + "e2e_ms": { + "mean": 1636.6455332696205, + "p50": 1650.3047684964258, + "p90": 1672.4332634243183, + "p95": 1708.9987661631312, + "p99": 1726.4511093095643 + }, + "ttft_ms": { + "mean": 337.84846669113904, + "p50": 368.39110546861775, + "p90": 501.928889635019, + "p95": 507.02435676357754, + "p99": 613.4727610013215 + }, + "tpot_ms": { + "mean": 10.27273139283467, + "p50": 9.955073681139336, + "p90": 12.024120250524309, + "p95": 12.093548784417461, + "p99": 12.511676546304457 + }, + "itl_ms": { + "mean": 10.269848071243864, + "p50": 9.954793488251148, + "p90": 12.023784178144144, + "p95": 12.093203529985034, + "p99": 12.511328950411581 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_128.jsonl" + }, + { + "name": "c8_i2048_o2048", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.54973888397217, + "request_throughput": 2.780203752489321, + "input_token_throughput": 5693.857285098129, + "output_token_throughput": 626.970698733248, + "total_token_throughput": 6320.827983831377, + "total_input_tokens": 327680, + "total_output_tokens": 36082, + "e2e_ms": { + "mean": 2571.8697952848743, + "p50": 2376.0124715045094, + "p90": 3531.4382432959974, + "p95": 4101.499575420167, + "p99": 5201.484706590415 + }, + "ttft_ms": { + "mean": 156.05718818696914, + "p50": 127.60109599912539, + "p90": 201.8362101982347, + "p95": 275.62024475191726, + "p99": 603.0409268877702 + }, + "tpot_ms": { + "mean": 10.757719886560922, + "p50": 10.89242498692186, + "p90": 11.446393503580977, + "p95": 11.598768451039856, + "p99": 11.99307822632247 + }, + "itl_ms": { + "mean": 10.702844962158807, + "p50": 10.82017472881797, + "p90": 11.400685704016238, + "p95": 11.540790897681546, + "p99": 11.943829748109716 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_2048.jsonl" + }, + { + "name": "c8_i2048_o256", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 52.18541706399992, + "request_throughput": 3.0659906349656425, + "input_token_throughput": 6279.148820409636, + "output_token_throughput": 604.0001550882316, + "total_token_throughput": 6883.1489754978675, + "total_input_tokens": 327680, + "total_output_tokens": 31520, + "e2e_ms": { + "mean": 2334.2623964919767, + "p50": 2330.2118405117653, + "p90": 2971.9119044952095, + "p95": 3004.2558109009406, + "p99": 3124.591278433217 + }, + "ttft_ms": { + "mean": 158.73426251928322, + "p50": 127.42687200079672, + "p90": 208.59998430823907, + "p95": 268.0313628050499, + "p99": 611.5358663420193 + }, + "tpot_ms": { + "mean": 11.131267257626003, + "p50": 11.17333754727414, + "p90": 11.92786350601456, + "p95": 12.152475340421605, + "p99": 12.446548674304928 + }, + "itl_ms": { + "mean": 11.069680648653042, + "p50": 11.113051173516164, + "p90": 11.841425316666124, + "p95": 12.086030156396962, + "p99": 12.403057491909903 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_256.jsonl" + }, + { + "name": "c8_i2048_o4096", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 59.21520890400279, + "request_throughput": 2.702008537357105, + "input_token_throughput": 5533.713484507351, + "output_token_throughput": 614.8251551222507, + "total_token_throughput": 6148.538639629602, + "total_input_tokens": 327680, + "total_output_tokens": 36407, + "e2e_ms": { + "mean": 2624.6837296948797, + "p50": 2390.2249354869127, + "p90": 3673.1235857703714, + "p95": 4094.224581937306, + "p99": 5661.190954251214 + }, + "ttft_ms": { + "mean": 164.46339333379, + "p50": 130.26967650512233, + "p90": 212.50965080107562, + "p95": 304.53456362884083, + "p99": 685.4601935023675 + }, + "tpot_ms": { + "mean": 10.838255479339688, + "p50": 10.899910423136323, + "p90": 11.584270936940777, + "p95": 11.765093191234756, + "p99": 11.979969078357305 + }, + "itl_ms": { + "mean": 10.793764061875354, + "p50": 10.857900357343716, + "p90": 11.548314111231585, + "p95": 11.743018766553384, + "p99": 11.925272947301288 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_4096.jsonl" + }, + { + "name": "c8_i2048_o512", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.762821699958295, + "request_throughput": 2.7699477845992333, + "input_token_throughput": 5672.85306285923, + "output_token_throughput": 605.804892665506, + "total_token_throughput": 6278.657955524736, + "total_input_tokens": 327680, + "total_output_tokens": 34993, + "e2e_ms": { + "mean": 2521.3216953932715, + "p50": 2340.7801749999635, + "p90": 3667.4506752635352, + "p95": 3808.3090486150477, + "p99": 4671.025379123165 + }, + "ttft_ms": { + "mean": 154.17553363658953, + "p50": 127.54814099753276, + "p90": 207.90915787802078, + "p95": 248.11484048841533, + "p99": 617.8795976977563 + }, + "tpot_ms": { + "mean": 10.860595376329208, + "p50": 10.945198745940369, + "p90": 11.707197147642002, + "p95": 11.808376366833382, + "p99": 12.001137625009958 + }, + "itl_ms": { + "mean": 10.805698258728144, + "p50": 10.888047823227195, + "p90": 11.661599113280404, + "p95": 11.776820527168061, + "p99": 11.951079170990065 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_512.jsonl" + }, + { + "name": "c8_i4096_o1024", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 64.92397661705036, + "request_throughput": 2.464420824740128, + "input_token_throughput": 10094.267698135563, + "output_token_throughput": 518.1290757714572, + "total_token_throughput": 10612.39677390702, + "total_input_tokens": 655360, + "total_output_tokens": 33639, + "e2e_ms": { + "mean": 2846.8765726112906, + "p50": 2693.235291488236, + "p90": 3686.392815841827, + "p95": 4602.444140485022, + "p99": 5170.714695255737 + }, + "ttft_ms": { + "mean": 256.3253793414333, + "p50": 200.92808597837575, + "p90": 408.3847037283703, + "p95": 442.47977496997936, + "p99": 1016.1522281781067 + }, + "tpot_ms": { + "mean": 12.360228898781688, + "p50": 12.529275761887078, + "p90": 13.544903270056238, + "p95": 13.768095517160951, + "p99": 14.462114144815686 + }, + "itl_ms": { + "mean": 12.290908117324742, + "p50": 12.467834575160465, + "p90": 13.514858231321586, + "p95": 13.711070186527868, + "p99": 14.255995922067301 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_1024.jsonl" + }, + { + "name": "c8_i4096_o128", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 46.47085372102447, + "request_throughput": 3.4430183047748133, + "input_token_throughput": 14102.602976357635, + "output_token_throughput": 440.039258214626, + "total_token_throughput": 14542.642234572262, + "total_input_tokens": 655360, + "total_output_tokens": 20449, + "e2e_ms": { + "mean": 2091.876247622349, + "p50": 2089.7147939831484, + "p90": 2107.6002722315025, + "p95": 2145.5566821066895, + "p99": 2225.516522292164 + }, + "ttft_ms": { + "mean": 707.0257856092212, + "p50": 675.6646229769103, + "p90": 1041.339729988249, + "p95": 1047.2809197148308, + "p99": 1071.5035220043499 + }, + "tpot_ms": { + "mean": 10.92299427938467, + "p50": 11.188700704653858, + "p90": 14.991371856230554, + "p95": 15.027174705320485, + "p99": 15.062308338558756 + }, + "itl_ms": { + "mean": 10.921993762760962, + "p50": 11.188328901615623, + "p90": 14.9909882158624, + "p95": 15.026840265321464, + "p99": 15.061849427742215 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_128.jsonl" + }, + { + "name": "c8_i4096_o2048", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 66.29023894300917, + "request_throughput": 2.413628349379683, + "input_token_throughput": 9886.221719059182, + "output_token_throughput": 513.3334943815076, + "total_token_throughput": 10399.555213440688, + "total_input_tokens": 655360, + "total_output_tokens": 34029, + "e2e_ms": { + "mean": 2865.3563139934704, + "p50": 2743.0460805189796, + "p90": 3883.7065048923246, + "p95": 4853.713436282124, + "p99": 6028.238608094397 + }, + "ttft_ms": { + "mean": 253.25430698540004, + "p50": 201.3168605044484, + "p90": 340.9121041302569, + "p95": 423.3174357010287, + "p99": 1022.5903978734273 + }, + "tpot_ms": { + "mean": 12.385675609766118, + "p50": 12.41990564159736, + "p90": 13.593211750988889, + "p95": 14.051944799220749, + "p99": 14.659802628899744 + }, + "itl_ms": { + "mean": 12.327093622960454, + "p50": 12.35750935978226, + "p90": 13.560235076824098, + "p95": 13.993455026303808, + "p99": 14.591190706431963 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_2048.jsonl" + }, + { + "name": "c8_i4096_o256", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 62.25946529000066, + "request_throughput": 2.5698903653401146, + "input_token_throughput": 10526.27093643311, + "output_token_throughput": 516.499777989013, + "total_token_throughput": 11042.770714422124, + "total_input_tokens": 655360, + "total_output_tokens": 32157, + "e2e_ms": { + "mean": 2789.2727036618453, + "p50": 2782.83986801398, + "p90": 3511.2314169178717, + "p95": 3625.363992372877, + "p99": 3833.623016313649 + }, + "ttft_ms": { + "mean": 256.85724185968866, + "p50": 200.55336799123324, + "p90": 421.0858193633612, + "p95": 457.12938645447144, + "p99": 1013.5523934691441 + }, + "tpot_ms": { + "mean": 12.6272613432991, + "p50": 12.802867527432571, + "p90": 13.773254597121301, + "p95": 14.045325650575263, + "p99": 14.568506100596926 + }, + "itl_ms": { + "mean": 12.558048593675748, + "p50": 12.695122376988785, + "p90": 13.6908571208121, + "p95": 13.99627949972378, + "p99": 14.511433540532495 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_256.jsonl" + }, + { + "name": "c8_i4096_o4096", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 64.2815860339906, + "request_throughput": 2.4890487287509635, + "input_token_throughput": 10195.143592963947, + "output_token_throughput": 543.8415906775308, + "total_token_throughput": 10738.985183641476, + "total_input_tokens": 655360, + "total_output_tokens": 34959, + "e2e_ms": { + "mean": 2921.1263272722135, + "p50": 2646.676472533727, + "p90": 4069.956865196581, + "p95": 5570.157847894009, + "p99": 8112.523980666883 + }, + "ttft_ms": { + "mean": 248.18690962783876, + "p50": 199.7099299915135, + "p90": 340.966394089628, + "p95": 462.5693424663051, + "p99": 1014.7484800923844 + }, + "tpot_ms": { + "mean": 12.268584562652679, + "p50": 12.345150528310768, + "p90": 13.420375906472373, + "p95": 13.678864128672437, + "p99": 14.315362949122035 + }, + "itl_ms": { + "mean": 12.203152687226083, + "p50": 12.302508034345959, + "p90": 13.362345801675275, + "p95": 13.600341813471513, + "p99": 14.240583484598446 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_4096.jsonl" + }, + { + "name": "c8_i4096_o512", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 65.25227290898329, + "request_throughput": 2.4520218663214224, + "input_token_throughput": 10043.481564452546, + "output_token_throughput": 538.7398543041455, + "total_token_throughput": 10582.221418756693, + "total_input_tokens": 655360, + "total_output_tokens": 35154, + "e2e_ms": { + "mean": 2931.060687192439, + "p50": 2709.595823456766, + "p90": 3999.5783280173773, + "p95": 5124.890309749753, + "p99": 6513.321161492495 + }, + "ttft_ms": { + "mean": 247.69492733612424, + "p50": 200.21844847360626, + "p90": 325.43213293538423, + "p95": 442.6875208271668, + "p99": 1015.5376534443344 + }, + "tpot_ms": { + "mean": 12.24628101497471, + "p50": 12.375387132825676, + "p90": 13.439804439564156, + "p95": 13.708735240033729, + "p99": 14.063654821676147 + }, + "itl_ms": { + "mean": 12.189888993120336, + "p50": 12.31609789090539, + "p90": 13.397326564884626, + "p95": 13.661249612179654, + "p99": 14.01123272006841 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_512.jsonl" + }, + { + "name": "c8_i512_o256", + "config": { + "concurrency": 8, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 52.72627985099098, + "request_throughput": 3.0345399002579705, + "input_token_throughput": 1553.684428932081, + "output_token_throughput": 629.4773705597628, + "total_token_throughput": 2183.161799491844, + "total_input_tokens": 81920, + "total_output_tokens": 33190, + "e2e_ms": { + "mean": 2369.8188398560887, + "p50": 2406.275746994652, + "p90": 2961.9523657194804, + "p95": 3039.6370284666773, + "p99": 3126.469637452392 + }, + "ttft_ms": { + "mean": 135.86247368766635, + "p50": 116.1904759646859, + "p90": 201.228951633675, + "p95": 271.8837070133306, + "p99": 289.0714289085008 + }, + "tpot_ms": { + "mean": 10.800091042884167, + "p50": 10.888139287493697, + "p90": 11.647774160941, + "p95": 11.890522297109356, + "p99": 12.06757466577572 + }, + "itl_ms": { + "mean": 10.746752771231769, + "p50": 10.832085453558161, + "p90": 11.590206155203498, + "p95": 11.81536591946112, + "p99": 12.02890745581324 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_512_256.jsonl" + }, + { + "name": "c128_i16384_o1024", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1401.2432732849848, + "request_throughput": 1.8269490022232189, + "input_token_throughput": 29932.732452425218, + "output_token_throughput": 453.17041808974545, + "total_token_throughput": 30385.90287051496, + "total_input_tokens": 41943040, + "total_output_tokens": 635002, + "e2e_ms": { + "mean": 69085.30075752884, + "p50": 59839.19886650983, + "p90": 112047.84775544539, + "p95": 135447.23007164142, + "p99": 185844.62699736594 + }, + "ttft_ms": { + "mean": 6827.033814014499, + "p50": 5296.662732493132, + "p90": 8734.584317711415, + "p95": 13719.54655988375, + "p99": 53845.72860275387 + }, + "tpot_ms": { + "mean": 254.24761383279738, + "p50": 264.87820864011053, + "p90": 266.3930666301598, + "p95": 266.592708826959, + "p99": 267.0054887261078 + }, + "itl_ms": { + "mean": 253.07364340401926, + "p50": 263.45366783234203, + "p90": 264.834891925602, + "p95": 264.927611077207, + "p99": 265.1289296694187 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_128_16384_1024.jsonl" + }, + { + "name": "c128_i16384_o2048", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1400.62394244998, + "request_throughput": 1.8277568463680782, + "input_token_throughput": 29945.968170894594, + "output_token_throughput": 468.4455121139211, + "total_token_throughput": 30414.413683008515, + "total_input_tokens": 41943040, + "total_output_tokens": 656116, + "e2e_ms": { + "mean": 69277.39991989332, + "p50": 59091.808564495295, + "p90": 114759.83570949172, + "p95": 141505.20173237895, + "p99": 189264.63393578478 + }, + "ttft_ms": { + "mean": 5874.331173899714, + "p50": 4223.068817984313, + "p90": 8200.802748126443, + "p95": 13475.141925734402, + "p99": 53934.113027165215 + }, + "tpot_ms": { + "mean": 249.3775254623817, + "p50": 254.3313727734818, + "p90": 266.0176135906007, + "p95": 266.260407924198, + "p99": 266.7113317080453 + }, + "itl_ms": { + "mean": 248.26982724391488, + "p50": 253.29337461046123, + "p90": 264.63717776394753, + "p95": 264.79745950692586, + "p99": 264.95209586103056 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_128_16384_2048.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 182.19263550196774, + "request_throughput": 0.10977392112967153, + "input_token_throughput": 14388.287390308307, + "output_token_throughput": 12.72279745892893, + "total_token_throughput": 14401.010187767235, + "total_input_tokens": 2621440, + "total_output_tokens": 2318, + "e2e_ms": { + "mean": 6192.48906809953, + "p50": 6180.761422525393, + "p90": 6213.763176015345, + "p95": 6221.419949684059, + "p99": 6221.434263539268 + }, + "ttft_ms": { + "mean": 5392.411926749628, + "p50": 5379.820790520171, + "p90": 5415.663412271533, + "p95": 5419.101967004826, + "p99": 5420.543884577928 + }, + "tpot_ms": { + "mean": 6.963251589076258, + "p50": 6.964099764229672, + "p90": 6.971392656594688, + "p95": 6.973061491507511, + "p99": 6.9767391713204505 + }, + "itl_ms": { + "mean": 6.962635385825456, + "p50": 6.963509944296697, + "p90": 6.9707969511412635, + "p95": 6.972460199486586, + "p99": 6.976156700761867 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 195.04452994198073, + "request_throughput": 0.10254068650861081, + "input_token_throughput": 13440.212862056636, + "output_token_throughput": 19.005916244371015, + "total_token_throughput": 13459.218778301007, + "total_input_tokens": 2621440, + "total_output_tokens": 3707, + "e2e_ms": { + "mean": 6686.716299253749, + "p50": 6740.412624028977, + "p90": 7094.8766680958215, + "p95": 7103.287010962958, + "p99": 7119.59390459524 + }, + "ttft_ms": { + "mean": 5397.539071351639, + "p50": 5400.779595016502, + "p90": 5420.498099375982, + "p95": 5431.664711123449, + "p99": 5440.220488613122 + }, + "tpot_ms": { + "mean": 6.998566515518891, + "p50": 7.004880447191741, + "p90": 7.022558806460007, + "p95": 7.030111121322866, + "p99": 7.030822165726561 + }, + "itl_ms": { + "mean": 6.937857768655009, + "p50": 6.934740034527012, + "p90": 6.970667251731075, + "p95": 6.977926703932042, + "p99": 6.977967237039455 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 201.48357103700982, + "request_throughput": 0.09926367642315745, + "input_token_throughput": 13010.688596136093, + "output_token_throughput": 20.185268600649067, + "total_token_throughput": 13030.87386473674, + "total_input_tokens": 2621440, + "total_output_tokens": 4067, + "e2e_ms": { + "mean": 6917.335643596016, + "p50": 6881.105848500738, + "p90": 7239.1232147521805, + "p95": 7364.33622519835, + "p99": 8109.9643394339355 + }, + "ttft_ms": { + "mean": 5500.612655151053, + "p50": 5505.672536994098, + "p90": 5544.737549085403, + "p95": 5555.720844605821, + "p99": 5562.254832139588 + }, + "tpot_ms": { + "mean": 7.003788900084021, + "p50": 7.004890883533367, + "p90": 7.0144188913927445, + "p95": 7.020465257247111, + "p99": 7.030904896468426 + }, + "itl_ms": { + "mean": 6.931159709011044, + "p50": 6.933711249570328, + "p90": 6.942321813002504, + "p95": 6.943215882998016, + "p99": 6.953637245724014 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_131072_512.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 426.9566505359835, + "request_throughput": 0.04684316305857477, + "input_token_throughput": 12279.654136827025, + "output_token_throughput": 8.190527060791798, + "total_token_throughput": 12287.844663887816, + "total_input_tokens": 5242880, + "total_output_tokens": 3497, + "e2e_ms": { + "mean": 14470.753410700127, + "p50": 14486.497411038727, + "p90": 14829.92560529965, + "p95": 14834.793809620896, + "p99": 14835.151171508478 + }, + "ttft_ms": { + "mean": 13218.703388556605, + "p50": 13146.110309491633, + "p90": 13440.408597118221, + "p95": 13504.541694695945, + "p99": 13586.495378156542 + }, + "tpot_ms": { + "mean": 7.217646949924548, + "p50": 7.205249976773752, + "p90": 7.285532500031017, + "p95": 7.323570450965531, + "p99": 7.362167690467771 + }, + "itl_ms": { + "mean": 7.0786326117089216, + "p50": 7.069641003355816, + "p90": 7.148613351776059, + "p95": 7.149781891644509, + "p99": 7.1514094155602415 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_262144_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 1098.2779603750096, + "request_throughput": 0.018210326275846376, + "input_token_throughput": 9547.455542510945, + "output_token_throughput": 1.6225400711779119, + "total_token_throughput": 9549.078082582122, + "total_input_tokens": 10485760, + "total_output_tokens": 1782, + "e2e_ms": { + "mean": 36907.33525130199, + "p50": 36869.80330350343, + "p90": 36962.52591308439, + "p95": 37308.40163663088, + "p99": 37807.7560897154 + }, + "ttft_ms": { + "mean": 36239.33900470438, + "p50": 36162.923870026134, + "p90": 36277.070767228724, + "p95": 36595.43939542782, + "p99": 37102.25245506095 + }, + "tpot_ms": { + "mean": 9.374547555038637, + "p50": 7.512703494884909, + "p90": 7.905592262867882, + "p95": 9.796026421859272, + "p99": 37.00170689786954 + }, + "itl_ms": { + "mean": 9.320936322930454, + "p50": 7.497354151818467, + "p90": 7.850911301760415, + "p95": 9.684160583476693, + "p99": 36.89945372730697 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_524288_128.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 114.83304778503953, + "request_throughput": 0.17416589027088075, + "input_token_throughput": 11414.135784792441, + "output_token_throughput": 39.37019949573259, + "total_token_throughput": 11453.505984288173, + "total_input_tokens": 1310720, + "total_output_tokens": 4521, + "e2e_ms": { + "mean": 4020.211619703332, + "p50": 3666.638041497208, + "p90": 4995.36795638851, + "p95": 5428.028983922561, + "p99": 7058.635749606762 + }, + "ttft_ms": { + "mean": 2462.7787359495414, + "p50": 2460.3703945176676, + "p90": 2469.169140793383, + "p95": 2475.2622663858347, + "p99": 2495.1480828761123 + }, + "tpot_ms": { + "mean": 6.921263477907232, + "p50": 6.920807158632908, + "p90": 6.927106867201686, + "p95": 6.928113336760625, + "p99": 6.932009929604653 + }, + "itl_ms": { + "mean": 6.869014879160817, + "p50": 6.86729956075302, + "p90": 6.892015080434886, + "p95": 6.89445152277582, + "p99": 6.902294996124454 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_1024.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 96.3714261049754, + "request_throughput": 0.20753039368966497, + "input_token_throughput": 13600.711880845884, + "output_token_throughput": 24.89327072307531, + "total_token_throughput": 13625.605151568958, + "total_input_tokens": 1310720, + "total_output_tokens": 2399, + "e2e_ms": { + "mean": 3313.691728649428, + "p50": 3332.6328665134497, + "p90": 3340.3004989027977, + "p95": 3344.6828095882665, + "p99": 3373.650279486319 + }, + "ttft_ms": { + "mean": 2497.0991775451694, + "p50": 2515.7667085004505, + "p90": 2525.6168330204673, + "p95": 2528.4263670502696, + "p99": 2558.934195797192 + }, + "tpot_ms": { + "mean": 6.865009487508972, + "p50": 6.865337037847244, + "p90": 6.8708567147718815, + "p95": 6.871842963009778, + "p99": 6.876171570825193 + }, + "itl_ms": { + "mean": 6.864437613271855, + "p50": 6.864770382355132, + "p90": 6.870263566796199, + "p95": 6.871242446588812, + "p99": 6.8754964826359855 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 105.9553989309934, + "request_throughput": 0.18875866828669668, + "input_token_throughput": 12370.488084836954, + "output_token_throughput": 32.66468754701286, + "total_token_throughput": 12403.152772383968, + "total_input_tokens": 1310720, + "total_output_tokens": 3461, + "e2e_ms": { + "mean": 3650.2073114039376, + "p50": 3586.5826304943766, + "p90": 4159.073864476522, + "p95": 4159.1440846154, + "p99": 4159.745420924155 + }, + "ttft_ms": { + "mean": 2460.456851005438, + "p50": 2457.9500810068566, + "p90": 2469.012875511544, + "p95": 2472.3303537844913, + "p99": 2483.1682235660264 + }, + "tpot_ms": { + "mean": 6.917922473178793, + "p50": 6.923507425054892, + "p90": 6.927156661085607, + "p95": 6.927327222851427, + "p99": 6.929477295310293 + }, + "itl_ms": { + "mean": 6.866275198167155, + "p50": 6.8652274387181045, + "p90": 6.89137497035926, + "p95": 6.892970948197751, + "p99": 6.895060622030097 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 108.81008856697008, + "request_throughput": 0.18380648580844106, + "input_token_throughput": 12045.941853941993, + "output_token_throughput": 34.57399998056776, + "total_token_throughput": 12080.515853922561, + "total_input_tokens": 1310720, + "total_output_tokens": 3762, + "e2e_ms": { + "mean": 3756.9655780505855, + "p50": 3748.656399984611, + "p90": 4199.485603865469, + "p95": 4315.5037806223845, + "p99": 4593.5078689147485 + }, + "ttft_ms": { + "mean": 2461.8244336452335, + "p50": 2458.6545474885497, + "p90": 2466.431661415845, + "p95": 2468.7689636222785, + "p99": 2498.347155104275 + }, + "tpot_ms": { + "mean": 6.922369205894936, + "p50": 6.921919020527464, + "p90": 6.926635313933737, + "p95": 6.92835883957081, + "p99": 6.928915238310461 + }, + "itl_ms": { + "mean": 6.865564523081028, + "p50": 6.869272967554027, + "p90": 6.883120653382063, + "p95": 6.885109553744558, + "p99": 6.890476212814103 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_512.jsonl" + }, + { + "name": "c32_i65536_o1024", + "config": { + "concurrency": 32, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 395.4516171090072, + "request_throughput": 0.40460069722232445, + "input_token_throughput": 26515.911293162255, + "output_token_throughput": 89.21951124623781, + "total_token_throughput": 26605.13080440849, + "total_input_tokens": 10485760, + "total_output_tokens": 35282, + "e2e_ms": { + "mean": 72731.84596702959, + "p50": 68012.98858498922, + "p90": 111259.59197211196, + "p95": 117936.92349772321, + "p99": 151338.19686311993 + }, + "ttft_ms": { + "mean": 18530.86177422847, + "p50": 15324.259108980186, + "p90": 36578.522250684895, + "p95": 54451.46387142016, + "p99": 68783.28264377428 + }, + "tpot_ms": { + "mean": 252.0000167805487, + "p50": 282.6309994048321, + "p90": 283.4716237898581, + "p95": 283.7050445944101, + "p99": 284.4103567225089 + }, + "itl_ms": { + "mean": 250.79581092386462, + "p50": 281.2108817644769, + "p90": 281.7336149556127, + "p95": 281.86471933110784, + "p99": 282.0100926023693 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_32_65536_1024.jsonl" + }, + { + "name": "c8_i131072_o512", + "config": { + "concurrency": 8, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 887.908358473971, + "request_throughput": 0.18019877667892276, + "input_token_throughput": 23619.014056859764, + "output_token_throughput": 33.12954509241995, + "total_token_throughput": 23652.143601952183, + "total_input_tokens": 20971520, + "total_output_tokens": 29416, + "e2e_ms": { + "mean": 41479.530511491976, + "p50": 40695.162369986065, + "p90": 58097.34920277842, + "p95": 66809.83657923935, + "p99": 79279.5650899608 + }, + "ttft_ms": { + "mean": 12820.608507814177, + "p50": 10825.261771999067, + "p90": 20686.60730908159, + "p95": 25478.948992854566, + "p99": 32945.62340407631 + }, + "tpot_ms": { + "mean": 156.62399342995585, + "p50": 157.0650420002514, + "p90": 211.635586992996, + "p95": 223.1889413108069, + "p99": 294.5085983403001 + }, + "itl_ms": { + "mean": 155.69947932134448, + "p50": 156.06387664204436, + "p90": 209.8603237579771, + "p95": 222.24342777714023, + "p99": 292.64473266343964 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_131072_512.jsonl" + }, + { + "name": "c8_i262144_o128", + "config": { + "concurrency": 8, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 2121.2106142269913, + "request_throughput": 0.07542862501577052, + "input_token_throughput": 19773.161476134148, + "output_token_throughput": 9.479963877763309, + "total_token_throughput": 19782.641440011914, + "total_input_tokens": 41943040, + "total_output_tokens": 20109, + "e2e_ms": { + "mean": 99110.04675216645, + "p50": 99465.57483251672, + "p90": 103144.93849380522, + "p95": 110875.081289644, + "p99": 129961.04960311025 + }, + "ttft_ms": { + "mean": 51391.65483183606, + "p50": 50290.368146525, + "p90": 53995.0529207359, + "p95": 61587.85037728085, + "p99": 80661.74531452528 + }, + "tpot_ms": { + "mean": 382.0441377238577, + "p50": 387.6563743937955, + "p90": 388.418829515108, + "p95": 388.6753990317422, + "p99": 389.2402474440457 + }, + "itl_ms": { + "mean": 381.6890187165073, + "p50": 387.61220860210665, + "p90": 388.2921533510144, + "p95": 388.5783935903698, + "p99": 388.8003312007251 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_262144_128.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 422.40927311300766, + "request_throughput": 0.37877956329144086, + "input_token_throughput": 24823.697459867868, + "output_token_throughput": 76.88988397589192, + "total_token_throughput": 24900.58734384376, + "total_input_tokens": 10485760, + "total_output_tokens": 32479, + "e2e_ms": { + "mean": 19699.146988233042, + "p50": 17983.775263506686, + "p90": 27382.01765177655, + "p95": 35998.2133779646, + "p99": 56237.927606889854 + }, + "ttft_ms": { + "mean": 4665.850258549472, + "p50": 4614.468439016491, + "p90": 7052.845655445708, + "p95": 8696.151139616268, + "p99": 14818.984791898976 + }, + "tpot_ms": { + "mean": 73.86492770566875, + "p50": 74.79552028993388, + "p90": 93.63153691253392, + "p95": 99.00013507767628, + "p99": 113.60191243889423 + }, + "itl_ms": { + "mean": 73.42846517454711, + "p50": 74.52521800299834, + "p90": 93.57217559080777, + "p95": 98.94534214206857, + "p99": 112.92381913901038 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 402.3544815850328, + "request_throughput": 0.3976592962745114, + "input_token_throughput": 26060.99964064638, + "output_token_throughput": 46.478915622685236, + "total_token_throughput": 26107.478556269067, + "total_input_tokens": 10485760, + "total_output_tokens": 18701, + "e2e_ms": { + "mean": 18805.63353704165, + "p50": 18806.962396978633, + "p90": 20208.182343194494, + "p95": 21397.725383506626, + "p99": 22483.11250662489 + }, + "ttft_ms": { + "mean": 8659.910049839164, + "p50": 9297.093524510274, + "p90": 12907.319448608905, + "p95": 13193.446341765231, + "p99": 15765.712861008938 + }, + "tpot_ms": { + "mean": 87.86875209645738, + "p50": 81.1571685254522, + "p90": 131.84152213743047, + "p95": 143.77713837975247, + "p99": 166.11118562007195 + }, + "itl_ms": { + "mean": 87.83150954708688, + "p50": 81.15654301274425, + "p90": 131.8411181340271, + "p95": 143.77673574197482, + "p99": 166.0579600479071 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_128.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 416.74979167600395, + "request_throughput": 0.38392340727164576, + "input_token_throughput": 25160.804418954576, + "output_token_throughput": 67.2513833475151, + "total_token_throughput": 25228.055802302093, + "total_input_tokens": 10485760, + "total_output_tokens": 28027, + "e2e_ms": { + "mean": 19445.902098786246, + "p50": 19457.487180974567, + "p90": 25653.07752349763, + "p95": 27248.316549730956, + "p99": 32918.893259076285 + }, + "ttft_ms": { + "mean": 5113.032919598118, + "p50": 4716.046326007927, + "p90": 9257.992207800271, + "p95": 10698.951332390427, + "p99": 14785.058989692472 + }, + "tpot_ms": { + "mean": 82.52785684036125, + "p50": 83.48555674335813, + "p90": 107.3290157292821, + "p95": 113.1300018626113, + "p99": 132.87966855654577 + }, + "itl_ms": { + "mean": 82.15261858594938, + "p50": 82.55044210092956, + "p90": 107.26047251854484, + "p95": 113.05118970898464, + "p99": 132.78087972009826 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_256.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 422.6590841940488, + "request_throughput": 0.37855568703816556, + "input_token_throughput": 24809.025505733218, + "output_token_throughput": 76.23401745035459, + "total_token_throughput": 24885.259523183573, + "total_input_tokens": 10485760, + "total_output_tokens": 32221, + "e2e_ms": { + "mean": 19689.273196897193, + "p50": 17639.091348508373, + "p90": 30207.329620694505, + "p95": 34072.42025379383, + "p99": 42942.271773040986 + }, + "ttft_ms": { + "mean": 4532.429489877904, + "p50": 4538.110625522677, + "p90": 7080.716864136048, + "p95": 9015.108168005827, + "p99": 14789.723641666926 + }, + "tpot_ms": { + "mean": 75.30331398641667, + "p50": 74.66872287350321, + "p90": 100.42571048979165, + "p95": 108.22462882297587, + "p99": 126.16376873402673 + }, + "itl_ms": { + "mean": 74.9958891435758, + "p50": 74.5155918840237, + "p90": 100.33134087725274, + "p95": 108.15634023765763, + "p99": 125.34686926273575 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_512.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh index 9120605..0e8adfa 100755 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh @@ -71,7 +71,6 @@ data["config"] = { "gpu_groups": "0,1,2,3,4,5,6,7", "kv_cache_dtype": "fp8", "block_size": 256, - "max_num_seqs": 256, "client": "bench_client.py", "lb_strategy": "round_robin", "scenarios": [ diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md deleted file mode 100644 index fc5aa6f..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# dsv4_h200_vllm_tp8_custom_bench_no_fp8 - -> **Note:** This variant does NOT use `--kv-cache-dtype fp8`. - -## 目的 - -当 vLLM 使用 **TP=8**（Tensor Parallel = 8）部署时，单张 H200 上运行 **1 个服务**，独占全部 8 张 GPU： - -| 服务 | 端口 | GPU 组 | -|------|------|--------| -| 1 | 30005 | 0, 1, 2, 3, 4, 5, 6, 7 | - -TP=8 提供最大的单服务 tensor-parallel 宽度，适合极高并发、大 batch 场景。本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API。 - -## 与多服务方案的区别 - -| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | TP=8 (1 服务) | -|------|---------------|---------------|---------------| -| 服务数量 | 4 | 2 | 1 | -| 每服务 GPU | 2 | 4 | 8 | -| 单服务吞吐 | 较低 | 中等 | 最高 | -| 集群总吞吐 | 相近 | 相近 | 相近 | -| 适用场景 | 低并发、低延迟 | 中等并发 | 极高并发、大 batch | - -## 文件说明 - -| 文件 | 说明 | -|------|------| -| `bench_client.py` | 自定义异步压测客户端（复用 TP=2/TP=4 版本） | -| `config.env` | 实验配置（单服务、端口 30005） | -| `run_bench.sh` | 编排脚本：启动 1 个服务 → 运行压测 → 解析结果 | -| `start_server.sh` | 启动 1 个 vLLM TP=8 服务（占用全部 8 GPU） | -| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | - -## 快速运行 - -```bash -# 完整流程（自动启动服务、压测、生成报告） -bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh - -# 跳过服务管理（已有服务在运行） -SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh - -# 单独运行压测客户端（单服务） -python3 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ - --output-file /tmp/test.jsonl - -# 解析已有结果 -python3 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py \ - experiments/dsv4_h200_vllm_tp8_custom_bench/results/ -``` - -## 场景设计 - -本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： - -| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | -|------|----------|----------|--------|------| -| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | -| 2 | 512 | 256 | 40 | 低并发 | -| 4 | 512 | 256 | 80 | 中低并发 | -| 8 | 512 | 256 | 160 | 中等并发 | -| 16 | 512 | 256 | 320 | 中高并发 | -| 32 | 512 | 256 | 640 | 高并发 | -| 64 | 512 | 256 | 1280 | 很高并发 | -| 128 | 512 | 256 | 2560 | 极限并发，测吞吐上限 | -| 1 | 2048 | 512 | 20 | 长输入基线 | -| 8 | 2048 | 512 | 160 | 长输入中并发 | -| 32 | 2048 | 512 | 640 | 长输入高并发 | -| 128 | 2048 | 512 | 2560 | 长输入极限并发 | -| 1 | 4096 | 1024 | 20 | 超长输入基线 | -| 8 | 4096 | 1024 | 160 | 超长输入中并发 | -| 32 | 4096 | 1024 | 640 | 超长输入高并发 | -| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | - -## 输出格式 - -同 TP=2/TP=4 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 - -## 单独启动/停止服务 - -```bash -# 启动服务 -bash experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh - -# 停止（run_bench.sh 会自动停止，但也可以手动） -port=30005 -pid_file="experiments/dsv4_h200_vllm_tp8_custom_bench/server_port${port}.pid" -[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true -rm -f "$pid_file" -pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true -``` - -## 注意事项 - -- 服务独占全部 8 张 GPU，确保无其他进程占用 GPU。 -- 模型文件路径：`/data/models/DeepSeek-V4-Flash` -- TP=8 相比 TP=2/TP=4 单服务延迟最低，但无法通过多服务并行提升低并发下的集群利用率。 -- 健康检查依赖 `/health` 端点（vLLM 默认启用）。 -- 如果服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py deleted file mode 100755 index d8edc36..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py +++ /dev/null @@ -1,617 +0,0 @@ -#!/usr/bin/env python3 -"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. - -Sends concurrent requests to multiple inference servers and records per-request -latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. - -Designed for TP=2 on 8-GPU setups where 4 independent services run on -(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via -round-robin or random load balancing. Unhealthy services are automatically -skipped with periodic retry. - -Usage: - # Single service (backward compatible) - python3 bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --output-file results.jsonl - - # Multi-service (4 services on ports 30005-30008) - python3 bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --lb-strategy round_robin \ - --output-file results.jsonl - -The output JSONL contains one object per request with raw timing data. -A final summary line ("completed" field) aggregates all requests. -""" - -import argparse -import asyncio -import json -import random -import sys -import time -import urllib.request -from dataclasses import asdict, dataclass, field -from typing import Any - -# Try aiohttp first; fall back to urllib for stdlib-only environments. -try: - import aiohttp - HAS_AIOHTTP = True -except ImportError: - HAS_AIOHTTP = False - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - -@dataclass -class RequestResult: - request_id: int - input_tokens: int = 0 - output_tokens: int = 0 - first_token_time_ms: float = 0.0 # TTFT - e2e_latency_ms: float = 0.0 # total wall time - inter_token_latencies: list[float] = field(default_factory=list) - success: bool = False - error: str = "" - target_port: int = 0 # which service handled this request - retry_count: int = 0 # how many times this request was retried - - @property - def tpot_ms(self) -> float: - """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" - if self.output_tokens <= 1: - return 0.0 - return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) - - @property - def mean_itl_ms(self) -> float: - if not self.inter_token_latencies: - return 0.0 - return sum(self.inter_token_latencies) / len(self.inter_token_latencies) - - def to_dict(self) -> dict[str, Any]: - d = asdict(self) - d["tpot_ms"] = self.tpot_ms - d["mean_itl_ms"] = self.mean_itl_ms - return d - - -# --------------------------------------------------------------------------- -# Health checker -# --------------------------------------------------------------------------- - -class HealthChecker: - """Periodically check service health and maintain a healthy-port list.""" - - def __init__( - self, - host: str, - ports: list[int], - check_interval: float = 5.0, - max_failures: int = 2, - recovery_interval: float = 10.0, - ): - self.host = host - self.all_ports = ports - self.check_interval = check_interval - self.max_failures = max_failures - self.recovery_interval = recovery_interval - - # port -> consecutive failure count - self._failures: dict[int, int] = {p: 0 for p in ports} - # port -> last healthy timestamp - self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} - self._lock = asyncio.Lock() - self._task: asyncio.Task | None = None - - async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: - url = f"http://{self.host}:{port}/health" - try: - if HAS_AIOHTTP and session is not None: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: - return resp.status == 200 - else: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - return resp.status == 200 - except Exception: - return False - - async def _check_all(self, session: aiohttp.ClientSession | None) -> None: - checks = [self._check_one(p, session) for p in self.all_ports] - results = await asyncio.gather(*checks, return_exceptions=True) - now = time.monotonic() - async with self._lock: - for port, ok in zip(self.all_ports, results): - if isinstance(ok, Exception): - ok = False - if ok: - self._failures[port] = 0 - self._last_healthy[port] = now - else: - self._failures[port] += 1 - - async def start(self, session: aiohttp.ClientSession | None) -> None: - async def _loop() -> None: - while True: - await asyncio.sleep(self.check_interval) - await self._check_all(session) - self._task = asyncio.create_task(_loop()) - - async def stop(self) -> None: - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - @property - async def healthy_ports(self) -> list[int]: - now = time.monotonic() - async with self._lock: - healthy = [] - for p in self.all_ports: - if self._failures[p] < self.max_failures: - healthy.append(p) - elif now - self._last_healthy[p] > self.recovery_interval: - # Give it another chance after recovery_interval. - self._failures[p] = max(0, self._failures[p] - 1) - healthy.append(p) - return healthy - - async def is_healthy(self, port: int) -> bool: - ports = await self.healthy_ports - return port in ports - - -# --------------------------------------------------------------------------- -# Load balancer with failover -# --------------------------------------------------------------------------- - -class LoadBalancer: - """Distribute requests across healthy backend ports with auto-failover.""" - - def __init__( - self, - ports: list[int], - health_checker: HealthChecker, - strategy: str = "round_robin", - ): - self.all_ports = ports - self.health_checker = health_checker - self.strategy = strategy - self._idx = 0 - self._lock = asyncio.Lock() - - def _next_round_robin(self, healthy: list[int]) -> int: - if not healthy: - return self.all_ports[self._idx % len(self.all_ports)] - for _ in range(len(self.all_ports)): - port = self.all_ports[self._idx % len(self.all_ports)] - self._idx += 1 - if port in healthy: - return port - # Fallback: all unhealthy, pick first healthy anyway. - return healthy[0] if healthy else self.all_ports[0] - - def _next_random(self, healthy: list[int]) -> int: - if healthy: - return random.choice(healthy) - return random.choice(self.all_ports) - - async def next_port(self) -> int: - healthy = await self.health_checker.healthy_ports - async with self._lock: - if self.strategy == "round_robin": - return self._next_round_robin(healthy) - elif self.strategy in ("random", "least_conn"): - return self._next_random(healthy) - else: - return self._next_round_robin(healthy) - - @property - def port_count(self) -> int: - return len(self.all_ports) - - -# --------------------------------------------------------------------------- -# Token helpers -# --------------------------------------------------------------------------- - -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: - tokens.extend(words) - return " ".join(tokens[:needed]) - - -# --------------------------------------------------------------------------- -# aiohttp-based async request (preferred) -# --------------------------------------------------------------------------- - -async def send_request_aiohttp( - session: aiohttp.ClientSession, - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - } - headers = {"Content-Type": "application/json"} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - first_token_received = False - last_token_time = 0.0 - - try: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status != 200: - text = await resp.text() - result.error = f"HTTP {resp.status}: {text[:200]}" - return result - - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or not line.startswith("data: "): - continue - data_str = line[len("data: "):] - if data_str == "[DONE]": - break - try: - chunk = json.loads(data_str) - except json.JSONDecodeError: - continue - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - now = time.perf_counter() - if not first_token_received: - result.first_token_time_ms = (now - start_time) * 1000 - first_token_received = True - else: - result.inter_token_latencies.append((now - last_token_time) * 1000) - last_token_time = now - result.output_tokens += 1 - - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.input_tokens = len(prompt.split()) # approximate - result.success = True - - except asyncio.TimeoutError: - result.error = "timeout" - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# urllib-based synchronous request (fallback, no aiohttp) -# --------------------------------------------------------------------------- - -def send_request_urllib( - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": False, - } - data = json.dumps(payload).encode("utf-8") - headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - - try: - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=600) as resp: - body = json.loads(resp.read().decode("utf-8")) - - choices = body.get("choices", []) - if not choices: - result.error = "no choices in response" - return result - - content = choices[0].get("message", {}).get("content", "") - result.output_tokens = len(content.split()) if content else 0 - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.first_token_time_ms = result.e2e_latency_ms - result.input_tokens = len(prompt.split()) - result.success = True - - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# Unified send_request wrapper with retry & failover -# --------------------------------------------------------------------------- - -async def send_request_with_retry( - session: aiohttp.ClientSession | None, - host: str, - lb: LoadBalancer, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, - max_retries: int = 2, -) -> RequestResult: - """Send a request, retrying on failure with a different port each time.""" - result = RequestResult(request_id=request_id) - for attempt in range(max_retries + 1): - port = await lb.next_port() - if HAS_AIOHTTP and session is not None: - result = await send_request_aiohttp( - session, host, port, model, request_id, prompt, max_tokens, temperature - ) - else: - loop = asyncio.get_event_loop() - result = await loop.run_in_executor( - None, send_request_urllib, - host, port, model, request_id, prompt, max_tokens, temperature - ) - result.retry_count = attempt - if result.success: - return result - # If failed, try another port on next iteration (unless last attempt). - if attempt < max_retries: - await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish - return result - - -# --------------------------------------------------------------------------- -# Benchmark orchestration -# --------------------------------------------------------------------------- - -async def run_benchmark( - host: str, - ports: list[int], - lb: LoadBalancer, - health_checker: HealthChecker, - model: str, - concurrency: int, - input_len: int, - output_len: int, - num_prompts: int, - temperature: float, - output_file: str, -) -> dict[str, Any]: - """Run the benchmark and write a JSONL file.""" - prompts = [make_prompt(input_len) for _ in range(num_prompts)] - results: list[RequestResult] = [] - semaphore = asyncio.Semaphore(concurrency) - - start = time.perf_counter() - - if HAS_AIOHTTP: - connector = aiohttp.TCPConnector(limit=concurrency * 2) - timeout = aiohttp.ClientTimeout(total=600, sock_read=300) - async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: - await health_checker.start(session) - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - session, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - else: - await health_checker.start(None) - from concurrent.futures import ThreadPoolExecutor - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - None, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - - duration = time.perf_counter() - start - - # Write JSONL - with open(output_file, "w", encoding="utf-8") as f: - for r in results: - f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") - - # Compute summary - successful = [r for r in results if r.success] - failed = len(results) - len(successful) - retried = [r for r in results if r.retry_count > 0] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - ttfts = [r.first_token_time_ms for r in successful] - tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] - e2es = [r.e2e_latency_ms for r in successful] - itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] - - total_input_tokens = sum(r.input_tokens for r in successful) - total_output_tokens = sum(r.output_tokens for r in successful) - - # Per-port breakdown - per_port_stats: dict[int, dict[str, Any]] = {} - for port in ports: - port_results = [r for r in successful if r.target_port == port] - port_failed = [r for r in results if r.target_port == port and not r.success] - if port_results or port_failed: - port_ttfts = [r.first_token_time_ms for r in port_results] - port_e2es = [r.e2e_latency_ms for r in port_results] - per_port_stats[port] = { - "count": len(port_results), - "failed": len(port_failed), - "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, - "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, - "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, - } - - # Health-check stats - healthy_at_end = await health_checker.healthy_ports - - summary = { - "completed": len(successful), - "failed": failed, - "retried": len(retried), - "duration": duration, - "request_throughput": len(successful) / duration if duration > 0 else 0.0, - "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, - "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, - "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, - "total_input_tokens": total_input_tokens, - "total_output_tokens": total_output_tokens, - "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, - "median_ttft_ms": percentile(ttfts, 50), - "p90_ttft_ms": percentile(ttfts, 90), - "p95_ttft_ms": percentile(ttfts, 95), - "p99_ttft_ms": percentile(ttfts, 99), - "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, - "median_tpot_ms": percentile(tpots, 50), - "p90_tpot_ms": percentile(tpots, 90), - "p95_tpot_ms": percentile(tpots, 95), - "p99_tpot_ms": percentile(tpots, 99), - "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, - "median_e2e_latency_ms": percentile(e2es, 50), - "p90_e2e_latency_ms": percentile(e2es, 90), - "p95_e2e_latency_ms": percentile(e2es, 95), - "p99_e2e_latency_ms": percentile(e2es, 99), - "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, - "median_itl_ms": percentile(itls, 50), - "p90_itl_ms": percentile(itls, 90), - "p95_itl_ms": percentile(itls, 95), - "p99_itl_ms": percentile(itls, 99), - "per_port_stats": per_port_stats, - "healthy_ports_at_end": healthy_at_end, - "lb_strategy": lb.strategy, - "num_services": lb.port_count, - } - - # Append summary as the last line - with open(output_file, "a", encoding="utf-8") as f: - f.write(json.dumps(summary, ensure_ascii=False) + "\n") - - return summary - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30005, - help="Single service port (backward compatible)") - parser.add_argument("--ports", default="", - help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") - parser.add_argument("--model", default="deepseek-v4-flash") - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--input-len", type=int, default=512) - parser.add_argument("--output-len", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=10) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--lb-strategy", default="round_robin", - choices=["round_robin", "random", "least_conn"], - help="Load balancing strategy across services") - parser.add_argument("--health-check-interval", type=float, default=5.0, - help="Seconds between health checks (default: 5)") - parser.add_argument("--health-max-failures", type=int, default=2, - help="Consecutive failures before marking a port unhealthy (default: 2)") - parser.add_argument("--health-recovery-interval", type=float, default=10.0, - help="Seconds before retrying an unhealthy port (default: 10)") - parser.add_argument("--request-max-retries", type=int, default=2, - help="Max retries per request on failure (default: 2)") - parser.add_argument("--output-file", required=True) - args = parser.parse_args() - - # Determine ports: --ports takes precedence, else fall back to --port. - if args.ports: - ports = [int(p.strip()) for p in args.ports.split(",")] - else: - ports = [args.port] - - health_checker = HealthChecker( - host=args.host, - ports=ports, - check_interval=args.health_check_interval, - max_failures=args.health_max_failures, - recovery_interval=args.health_recovery_interval, - ) - lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) - - summary = asyncio.run(run_benchmark( - host=args.host, - ports=ports, - lb=lb, - health_checker=health_checker, - model=args.model, - concurrency=args.concurrency, - input_len=args.input_len, - output_len=args.output_len, - num_prompts=args.num_prompts, - temperature=args.temperature, - output_file=args.output_file, - )) - - print(json.dumps(summary, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env deleted file mode 100644 index 618099b..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env +++ /dev/null @@ -1,83 +0,0 @@ -# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench -# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. -# -# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: -# Service 1: port 30005, GPUs 0,1 -# Service 2: port 30006, GPUs 2,3 -# Service 3: port 30007, GPUs 4,5 -# Service 4: port 30008, GPUs 6,7 -# -# bench_client.py distributes requests across all 4 services via round-robin -# load balancing, giving accurate cluster-wide throughput even at low -# per-service concurrency. - -EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench_no_fp8" -MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" - -# Primary port (backward compatible); multi-service ports are in PORTS. -PORT="30005" -PORTS="30005" - -BACKEND="vllm" -ENGINE="vllm" - -# Native virtual environments on the host. -VENV_SERVER="/data/user1/yy/envs/vllm" -VENV_CLIENT="/data/user1/yy/envs/vllm" - -# Load balancing strategy: round_robin | random | least_conn -LB_STRATEGY="round_robin" - -# Benchmark scenarios: "concurrency input_len output_len num_prompts". -# For single-service TP=8 we cover standard concurrency gradients. -SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" -) - -# Server start script bundled with this experiment. -SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py deleted file mode 100755 index eea1a2e..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. - -Reads JSONL files produced by bench_client.py (one request per line, -last line is the summary) and generates: - - results.json (appended scenarios, following the standard schema) - - report.md (human-readable summary) - -Usage: - python3 parse_results.py -""" - -import json -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: - """Read the summary JSON (last line) and all per-request lines.""" - requests: list[dict] = [] - summary: dict | None = None - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - # The summary line has "completed" and "duration" keys. - if "completed" in obj and "duration" in obj: - summary = obj - else: - requests.append(obj) - return summary, requests - - -def compute_metrics(summary: dict, requests: list[dict]) -> dict: - completed = summary.get("completed", 0) - total = len(requests) - failed = total - completed if total > 0 else 0 - duration_s = summary.get("duration", 0.0) - - # Extract per-request metrics for percentiles. - ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] - tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] - e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] - itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": summary.get("request_throughput", 0.0), - "input_token_throughput": summary.get("input_throughput", 0.0), - "output_token_throughput": summary.get("output_throughput", 0.0), - "total_token_throughput": summary.get("total_throughput", 0.0), - "total_input_tokens": summary.get("total_input_tokens", 0), - "total_output_tokens": summary.get("total_output_tokens", 0), - "e2e_ms": { - "mean": summary.get("mean_e2e_latency_ms", 0.0), - "p50": percentile(e2es, 50), - "p90": percentile(e2es, 90), - "p95": percentile(e2es, 95), - "p99": percentile(e2es, 99), - }, - "ttft_ms": { - "mean": summary.get("mean_ttft_ms", 0.0), - "p50": percentile(ttfts, 50), - "p90": percentile(ttfts, 90), - "p95": percentile(ttfts, 95), - "p99": percentile(ttfts, 99), - }, - "tpot_ms": { - "mean": summary.get("mean_tpot_ms", 0.0), - "p50": percentile(tpots, 50), - "p90": percentile(tpots, 90), - "p95": percentile(tpots, 95), - "p99": percentile(tpots, 99), - }, - "itl_ms": { - "mean": summary.get("mean_itl_ms", 0.0), - "p50": percentile(itls, 50), - "p90": percentile(itls, 90), - "p95": percentile(itls, 95), - "p99": percentile(itls, 99), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: - ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms - tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") - f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") - f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") - f.write(f"- **Result root**: `{result_root}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - - summary, requests = parse_jsonl(jsonl_path) - if summary is None: - continue - - metrics = compute_metrics(summary, requests) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": "random", - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - for s in scenarios: - append_scenario(results_json, s) - - generate_report(result_root, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh deleted file mode 100755 index 2319cae..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env bash -# Custom benchmark orchestrator for single-service vLLM TP=8 on 8x H200. -# -# This runs 1 vLLM service (TP=8) on all 8 GPUs (0,1,2,3,4,5,6,7). -# bench_client.py sends requests to the single service. -# -# Usage: -# bash run_bench.sh -# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" - -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -ensure_result_root "$RESULT_ROOT" -log_init "${LOG_DIR}/orchestrator.log" - -log "experiment=${EXPERIMENT_NAME}" -log "run_id=${RUN_ID}" -log "result_root=${RESULT_ROOT}" -log "platform=${PLATFORM}" -log "chip=${CHIP}" -log "accelerator=${ACCELERATOR}" -log "engine=${ENGINE}" -log "hardware=${HARDWARE}" -log "model=${MODEL_PATH}" -log "services=${PORTS}" -log "lb_strategy=${LB_STRATEGY}" -log "server_start_script=${SERVER_START_SCRIPT}" - -# Write initial metadata for this run. -METADATA_JSON="${RESULT_ROOT}/results.json" -write_metadata_json \ - "$METADATA_JSON" \ - "$EXPERIMENT_NAME" \ - "$RUN_ID" \ - "$MODEL_PATH" \ - "$BACKEND" \ - "$ENGINE" \ - "$HARDWARE" \ - "$ACCELERATOR" \ - "$CHIP" \ - "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_SERVER" \ - "H200 1x vLLM TP=8 single-service (no FP8 KV cache) benchmark using bench_client.py" - -# Update metadata with config. -"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' -import json, sys -path = sys.argv[1] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 8, - "num_services": 1, - "ports": "30005", - "gpu_groups": "0,1,2,3,4,5,6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", "2 512 256 40", "4 512 256 80", - "8 512 256 160", "16 512 256 320", "32 512 256 640", - "64 512 256 1280", "128 512 256 2560", - "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", - "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" - ] -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY - -# --------------------------------------------------------------------------- -# Server helpers (single-service) -# --------------------------------------------------------------------------- - -is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -are_all_services_healthy() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - for p in "${PORT_LIST[@]}"; do - if ! is_server_healthy "$p"; then - return 1 - fi - done - return 0 -} - -stop_all_servers() { - log "stopping all vllm services" - # Kill by PID files first. - for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do - [[ -f "$pid_file" ]] || continue - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "killing pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - done - # Fallback: kill any remaining vllm processes for this model. - pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true - sleep 2 -} - -start_all_servers() { - log "starting all vllm services with ${SERVER_START_SCRIPT}" - if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" - exit 1 - fi - bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - - if ! are_all_services_healthy "$PORTS"; then - log "error: not all vllm services became healthy" - exit 1 - fi - log "all vllm services are healthy" -} - -# --------------------------------------------------------------------------- -# Pre-benchmark health check with detailed reporting -# --------------------------------------------------------------------------- - -check_services_detailed() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy_count=0 - local unhealthy_ports="" - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - ((healthy_count++)) - else - unhealthy_ports="${unhealthy_ports}${p} " - fi - done - echo "$healthy_count" - if [[ -n "$unhealthy_ports" ]]; then - echo "unhealthy: ${unhealthy_ports}" >&2 - fi -} - -wait_for_all_services() { - local ports="$1" - local max_wait="${2:-240}" - local interval="${3:-5}" - local elapsed=0 - while (( elapsed < max_wait )); do - local healthy_count - healthy_count="$(check_services_detailed "$ports" | head -1)" - IFS=',' read -ra PORT_LIST <<< "$ports" - if (( healthy_count == ${#PORT_LIST[@]} )); then - return 0 - fi - log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" - sleep "$interval" - ((elapsed += interval)) - done - return 1 -} - -# --------------------------------------------------------------------------- -# Mid-benchmark health check (called between scenarios) -# --------------------------------------------------------------------------- - -check_and_report_services() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy=() - local unhealthy=() - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - healthy+=("$p") - else - unhealthy+=("$p") - fi - done - log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" - if [[ ${#unhealthy[@]} -gt 0 ]]; then - log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" - fi - return ${#unhealthy[@]} -} - -on_exit() { - local code=$? - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_all_servers - fi - exit "$code" -} -trap on_exit EXIT - -if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming services are already running" - if ! wait_for_all_services "$PORTS"; then - log "error: not all healthy services found on ports ${PORTS}" - exit 1 - fi -else - stop_all_servers - start_all_servers -fi - -log "===== BENCHMARK START =====" - -for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len num_prompts <<< "$scenario" - # Fallback for legacy 3-field scenarios. - if [[ -z "${num_prompts:-}" ]]; then - num_prompts="$NUM_PROMPTS" - fi - - output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" - - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ - --host 127.0.0.1 \ - --port "$PORT" \ - --model "$SERVED_MODEL_NAME" \ - --concurrency "$concurrency" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num-prompts "$num_prompts" \ - --output-file "$output_file" \ - > "$detail_log" 2>&1 || { - log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" - # After failure, check service health before continuing. - check_and_report_services "$PORTS" - continue - } - - log "finished scenario: output=${output_file}" - - # Between scenarios, report service health so we can spot degradation early. - check_and_report_services "$PORTS" || true -done - -log "===== BENCHMARK DONE =====" - -log "parsing results" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { - log "WARNING: parser failed; see ${LOG_DIR}/parse.log" -} - -log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh deleted file mode 100755 index b9dad2f..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# Start 1 vLLM TP=8 service on 8x H200. -# Uses all 8 GPUs: (0,1,2,3,4,5,6,7). -# This maximizes single-service throughput for highest-concurrency workloads. -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -VENV="${VENV_SERVER}" -export PATH="$VENV/bin:$PATH" -VLLM="$VENV/bin/vllm" - -TP=8 -LOG_DIR="${SCRIPT_DIR}/logs" -mkdir -p "$LOG_DIR" - -# 1 service on all 8 GPUs. -# Port and GPU mapping. -SERVICES=( - "30005:0,1,2,3,4,5,6,7" -) - -# Kill any existing vLLM processes for this model. -pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true -sleep 2 - -# Remove old PID files. -rm -f "${SCRIPT_DIR}"/*.pid - -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - GPUS="${svc##*:}" - LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - - echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" - echo "Log: ${LOG}" - - CUDA_VISIBLE_DEVICES="${GPUS}" \ - nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --block-size 256 \ - --served-model-name "$SERVED_MODEL_NAME" \ - --port "$PORT" \ - > "$LOG" 2>&1 & - PID=$! - echo $PID > "$PID_FILE" - echo "PID: $PID" -done - -echo "" -echo "Waiting for service to become healthy..." - -MAX_WAIT=240 -all_healthy=0 -for i in $(seq 1 $MAX_WAIT); do - all_healthy=1 - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then - all_healthy=0 - break - fi - done - - if [[ "$all_healthy" -eq 1 ]]; then - echo "Service is healthy!" - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - echo " - http://127.0.0.1:${PORT}" - done - exit 0 - fi - - # Check if any process died early. - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - if [[ -f "$PID_FILE" ]]; then - PID="$(cat "$PID_FILE")" - if ! kill -0 "$PID" 2>/dev/null; then - echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - tail -200 $LOG 2>/dev/null || true - exit 1 - fi - fi - done - - if (( i % 10 == 0 )); then - echo "Waiting... (${i}/${MAX_WAIT})" - fi - sleep 5 -done - -echo "ERROR: Service did not become healthy after ${MAX_WAIT} retries" -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - echo "--- Last 50 lines of port ${PORT} ---" - tail -50 $LOG 2>/dev/null || true -done -exit 1 From d904d663a41e9dc130f814f44f08b4d49c863398 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 13 Jul 2026 02:59:33 +0000 Subject: [PATCH 45/53] Add dsv4_pro6000_vllm_tp_dp_matrix experiment for RTX 6000D - vLLM adaptive concurrency benchmark adapted from H200 version - Key changes: - Model path: /data/hf_models/DeepSeek-V4-Flash - Venv path: /root/.miniconda3/envs/sglang - Dataset path: /data/yy/sskj/datasets/ - DSL renamed to OSL (Output Sequence Length) - Includes: config.env, adaptive_config.env, run_adaptive_concurrency.sh, start_vllm_dp.sh, start_vllm_docker.sh --- .../adaptive_config.env | 49 ++++++ .../dsv4_pro6000_vllm_tp_dp_matrix/config.env | 76 ++++++++ .../run_adaptive_concurrency.sh | 164 ++++++++++++++++++ .../start_vllm_docker.sh | 103 +++++++++++ .../start_vllm_dp.sh | 87 ++++++++++ 5 files changed, 479 insertions(+) create mode 100644 experiments/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env create mode 100644 experiments/dsv4_pro6000_vllm_tp_dp_matrix/config.env create mode 100755 experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100755 experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh create mode 100755 experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh diff --git a/experiments/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..1eda1f0 --- /dev/null +++ b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_vllm_tp_dp_matrix/config.env b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/config.env new file mode 100644 index 0000000..228a6e4 --- /dev/null +++ b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/config.env @@ -0,0 +1,76 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash on RTX 6000D (8 GPUs) using vLLM. +# Tests vLLM with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_pro6000_vllm_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +VLLM_PORT="${VLLM_PORT:-30030}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# vLLM server settings +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +BLOCK_SIZE="${BLOCK_SIZE:-256}" + +# Deployment switch. 0 = native vllm venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" + +# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving, +# so the client runs inside the SGLang image and targets the vLLM backend via the +# OpenAI-compatible API. +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline nodes. +DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..ca23f6a --- /dev/null +++ b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh new file mode 100755 index 0000000..e133fd7 --- /dev/null +++ b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Start vLLM server in Docker for a given TPxDP configuration. +# Usage: start_vllm_docker.sh +# +# Uses the vllm/vllm-openai image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +PORT="${VLLM_PORT:-30030}" +NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: vllm ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --ulimit memlock=-1 \ + --entrypoint vllm \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh new file mode 100755 index 0000000..741ca9c --- /dev/null +++ b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Start vLLM server for a given TP×DP configuration. +# Usage: start_vllm_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the +# local VENV_VLLM environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${VLLM_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 From 819850c4a1e8a96d09dcdb114550ca961459a79d Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 13 Jul 2026 03:04:46 +0000 Subject: [PATCH 46/53] Update README.md with new experiments and RTX 6000D platform - Add all H200 experiments: TP/DP matrix, adaptive concurrency, custom bench, MTP vs default, long context matrix, 64k/256k probes - Add RTX 6000D platform and experiments (sglang/vllm TP/DP adaptive) - Update environment requirements section with RTX 6000D details - Update platforms directory description to include RTX 6000D --- README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 80832b5..132b23b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ | 目录/文件 | 说明 | |---|---| -| `platforms/` | 芯片/加速器平台配置（`*.env`），如 P800、H200 | +| `platforms/` | 芯片/加速器平台配置（`*.env`），如 P800、H200、RTX 6000D | | `scripts/common/` | 跨实验复用的 orchestration 组件（server 启停、health check、元数据生成、结果解析） | | `scripts/` | 仅保留仍在使用的 legacy DSpark 脚本与 SLO 标准 | | `experiments/` | 以实验为单位的自包含目录（脚本 + 配置 + 结果） | @@ -30,6 +30,19 @@ | DSV4 H200 SGLang vs vLLM | `experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh` | NVIDIA H200 上 SGLang 与 vLLM 控制变量对比（TP=8，最长 200k 上下文） | | DSV4 H200 vLLM DSpark vs default | `experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh` | vLLM 开启 DSpark 投机解码 vs 默认配置，验证 TTFT 差异 | | DSV4 H200 max context length | `experiments/dsv4_h200_max_context_length/run_bench.sh` | 探索 SGLang / vLLM 在 H200 上能支持的最大输入长度 | +| DSV4 H200 SGLang TP/DP Matrix | `experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh` | H200 上 SGLang 不同 TP×DP 配置的固定并发矩阵测试 | +| DSV4 H200 SGLang TP/DP Adaptive | `experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh` | H200 上 SGLang 不同 TP×DP 配置的自适应并发饱和点搜索 | +| DSV4 H200 vLLM TP/DP Matrix | `experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh` | H200 上 vLLM 不同 TP×DP 配置的固定并发矩阵测试 | +| DSV4 H200 vLLM TP/DP Adaptive | `experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh` | H200 上 vLLM 不同 TP×DP 配置的自适应并发饱和点搜索 | +| DSV4 H200 64k SGLang vs vLLM | `experiments/dsv4_h200_64k_sglang_vs_vllm/run_bench.sh` | H200 上 64k 上下文 SGLang 与 vLLM 对比 | +| DSV4 H200 256k 4k Probe | `experiments/dsv4_h200_256k_4k_probe/run_bench.sh` | H200 上 256k 输入/4k 输出长上下文探测 | +| DSV4 H200 vLLM TP2 Custom Bench | `experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh` | H200 上 vLLM TP=2 自定义压测客户端（多服务负载均衡） | +| DSV4 H200 vLLM TP4 Custom Bench | `experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh` | H200 上 vLLM TP=4 自定义压测客户端 | +| DSV4 H200 vLLM TP8 Custom Bench | `experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh` | H200 上 vLLM TP=8 自定义压测客户端 | +| DSV4 H200 vLLM MTP vs Default | `experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh` | H200 上 vLLM MTP（Multi-Token Prediction）vs 默认配置对比 | +| DSV4 H200 Long Context Matrix | `experiments/dsv4_h200_long_context_matrix/run_bench.sh` | H200 上长上下文矩阵测试（多后端对比） | +| **DSV4 RTX 6000D SGLang TP/DP Adaptive** | `experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh` | **RTX 6000D 上 SGLang 自适应并发饱和点搜索** | +| **DSV4 RTX 6000D vLLM TP/DP Adaptive** | `experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh` | **RTX 6000D 上 vLLM 自适应并发饱和点搜索** | ### 旧结构（scripts/ + bench_results/） @@ -72,10 +85,72 @@ bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh bash experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh ``` -### H200 max context length +### H200 SGLang TP/DP Matrix ```bash -bash experiments/dsv4_h200_max_context_length/run_bench.sh +bash experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +``` + +### H200 SGLang TP/DP 自适应并发搜索 + +```bash +bash experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +``` + +### H200 vLLM TP/DP Matrix + +```bash +bash experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +``` + +### H200 vLLM TP/DP 自适应并发搜索 + +```bash +bash experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +``` + +### H200 64k SGLang vs vLLM + +```bash +bash experiments/dsv4_h200_64k_sglang_vs_vllm/run_bench.sh +``` + +### H200 256k 4k Probe + +```bash +bash experiments/dsv4_h200_256k_4k_probe/run_bench.sh +``` + +### H200 vLLM TP2/TP4/TP8 Custom Bench + +```bash +bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh +bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh +bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh +``` + +### H200 vLLM MTP vs Default + +```bash +bash experiments/dsv4_h200_vllm_mtp_vs_default/run_bench.sh +``` + +### H200 Long Context Matrix + +```bash +bash experiments/dsv4_h200_long_context_matrix/run_bench.sh +``` + +### RTX 6000D SGLang TP/DP 自适应并发搜索 + +```bash +bash experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +``` + +### RTX 6000D vLLM TP/DP 自适应并发搜索 + +```bash +bash experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh ``` ### DSpark grid / spec-tokens（旧结构） @@ -109,6 +184,7 @@ python3 experiments/dsv4_h200_dspark/parse_results.py \ ## 环境要求 - **NVIDIA H200**：Python env `/data/user1/yy/envs/vllm-dspark`（服务端）、`/data/user1/yy/envs/sglang`（压测客户端）；模型 `/data/models/DeepSeek-V4-Flash`、`/data/models/DeepSeek-V4-Flash-DSpark`。 +- **NVIDIA RTX 6000D**：Python env `/root/.miniconda3/envs/sglang`（压测客户端）；模型 `/data/hf_models/DeepSeek-V4-Flash`；Docker 镜像 `lmsysorg/sglang:latest`、`vllm/vllm-openai:latest`。 - **Kunlun P800**：Docker 镜像 `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202`；模型 `/data1/models/DeepSeek-V4-Flash-INT8`；压测客户端在容器内运行。 平台相关常量见 `platforms/*.env`。 From a7f453b9979df1b31223a8ad580b97113884607e Mon Sep 17 00:00:00 2001 From: Root Date: Mon, 13 Jul 2026 05:22:31 +0000 Subject: [PATCH 47/53] feat: add run_bench.sh and fix config for dsv4_pro6000_sglang_tp_dp_matrix - Migrate run_bench.sh from dsv4_h200_sglang_tp_dp_matrix - Add missing generate_scenarios.py, compare.py, matrix.json - Fix MODEL_PATH to /root/data/models/ds_flash_v4 - Fix DATASET_PATH to local dummy_sharegpt.json - Tune CONTEXT_LENGTH, MAX_RUNNING_REQUESTS, MEM_FRACTION_STATIC for RTX 6000D 83GB - Exclude GPU 2 due to DRAM Uncorrectable ECC hardware error --- .../compare.py | 162 +++++ .../config.env | 14 +- .../generate_scenarios.py | 98 +++ .../matrix.json | 81 +++ .../run_bench.sh | 558 ++++++++++++++++++ 5 files changed, 906 insertions(+), 7 deletions(-) create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/compare.py create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/compare.py b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/compare.py new file mode 100755 index 0000000..2029e7b --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h200_sglang_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: SGLang (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, dsl = re.findall(r"\d+", scenario_name) + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/DSL. + f.write("\n## Best throughput per (ISL, DSL)\n\n") + f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, dsl = re.findall(r"\d+", scenario_name) + isl_i, dsl_i = int(isl), int(dsl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]: + best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s) + for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env index c963f5c..9521ce9 100644 --- a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -7,7 +7,7 @@ EXPERIMENT="dsv4_pro6000_sglang_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash" +MODEL_PATH="/root/data/models/ds_flash_v4" SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30031}" @@ -20,7 +20,7 @@ VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" # Run the benchmark client natively (0) or inside Docker (1). USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" -export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,3,4,5,6,7}" # Runtime working directory for logs, pid files, and tmp. Defaults to a local # directory under this experiment so the benchmark is self-contained. @@ -33,10 +33,10 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) -# SGLang server settings -CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" +# SGLang server settings (tuned for RTX 6000D 83GB) +CONTEXT_LENGTH="${CONTEXT_LENGTH:-262144}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-32}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" # Deployment switch. 0 = native sglang venv, 1 = Docker. @@ -46,7 +46,7 @@ DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" # Dataset used by sglang.bench_serving --dataset-name random. # The random sampler needs a ShareGPT-style JSON file locally; it falls back to # downloading from HuggingFace, which usually fails on offline nodes. -DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" +DATASET_PATH="${DATASET_PATH:-/root/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/generate_scenarios.py new file mode 100755 index 0000000..f722479 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return only the low and high concurrency values in [low, high]. + + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if low == high: + return [low] + return [low, high] + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + dsl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for dsl_str in sorted(dsl_map.keys(), key=int): + mark = dsl_map[dsl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(dsl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..55dcf58 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json @@ -0,0 +1,81 @@ +{ + "comment": "ISL/DSL matrix for dsv4_h200_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 }, + "262144": { "low": 1, "high": 2 }, + "524288": { "low": 1, "high": 2 }, + "1048576": { "low": 1, "high": 2 } + } +} diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..0671523 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh @@ -0,0 +1,558 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash on SGLang (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + # In container-reuse mode, pid is the server PID inside the container. + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server inside container (tp=${tp}, dp=${dp}, pid=${pid})" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + else + if kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + fi + rm -f "$pid_file" + fi + + # Fallback: remove any Docker container started by this experiment (legacy mode). + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + + # Fallback: kill any sglang serve processes for this model. + pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "sglang serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + + local args=( + "sglang serve" "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --dp-size "$dp" + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + fi + + if ! is_server_healthy; then + log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}" + return 1 + fi + log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") + docker run --rm \ + --network host \ + "${vol_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "${DOCKER_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} +export -f run_bench_serving + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${dsl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "sglang", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_sglang_dp.sh", + "server_args": $args + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl dsl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${dsl}" + output_file="${raw_dir}/sglang_main_${conc}_${isl}_${dsl}.jsonl" + detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${dsl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$dsl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ + --request-rate 10000 \ + --output-file "'"$output_file"'" \ + --output-details \ + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" From 60826dda0950212c36040eb397268c4ccba1b7ee Mon Sep 17 00:00:00 2001 From: iiGray Date: Mon, 13 Jul 2026 17:26:33 +0800 Subject: [PATCH 48/53] add run_adaptive_concurrency_add16 --- .../run_adaptive_concurrency_add16.sh | 184 ++++++++++++++++++ .../run_adaptive_concurrency_add16.sh | 167 ++++++++++++++++ .../run_adaptive_concurrency_add16.sh | 184 ++++++++++++++++++ .../run_adaptive_concurrency_add16.sh | 167 ++++++++++++++++ scripts/common/adaptive_bench_lib.sh | 29 ++- 5 files changed, 725 insertions(+), 6 deletions(-) create mode 100755 experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh create mode 100755 experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh create mode 100755 experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..0e6801f --- /dev/null +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local dsl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$dsl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..9acbe5a --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/DSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local dsl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$dsl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..b9b28f3 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..482886d --- /dev/null +++ b/experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index 3f8b158..c64f4f5 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -29,7 +29,7 @@ export -f adaptive_warmup_request_count adaptive_validate_config() { local name - for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY SEARCH_MULTIPLIER \ + for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY \ NUM_PROMPTS_MULTIPLIER PLATEAU_PATIENCE MAX_POINT_RETRIES \ BENCH_WARMUP_MAX_REQUESTS; do if ! [[ "${!name}" =~ ^[0-9]+$ ]]; then @@ -45,9 +45,16 @@ adaptive_validate_config() { log "ERROR: SEARCH_MAX_CONCURRENCY must be >= SEARCH_START_CONCURRENCY" return 1 fi - if (( SEARCH_MULTIPLIER < 2 )); then - log "ERROR: SEARCH_MULTIPLIER must be at least 2" - return 1 + if [[ -n "${SEARCH_ADDEND:-}" ]]; then + if ! [[ "${SEARCH_ADDEND}" =~ ^[0-9]+$ ]] || (( SEARCH_ADDEND < 1 )); then + log "ERROR: SEARCH_ADDEND must be at least 1" + return 1 + fi + else + if ! [[ "${SEARCH_MULTIPLIER:-}" =~ ^[0-9]+$ ]] || (( SEARCH_MULTIPLIER < 2 )); then + log "ERROR: SEARCH_MULTIPLIER must be at least 2" + return 1 + fi fi if (( NUM_PROMPTS_MULTIPLIER < 1 || PLATEAU_PATIENCE < 1 )); then log "ERROR: NUM_PROMPTS_MULTIPLIER and PLATEAU_PATIENCE must be at least 1" @@ -444,7 +451,12 @@ adaptive_run_shape() { stop_reason="SEARCH_CAP_REACHED" break fi - local next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + local next_concurrency + if [[ -n "${SEARCH_ADDEND:-}" ]]; then + next_concurrency=$((concurrency + SEARCH_ADDEND)) + else + next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + fi if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then next_concurrency="$SEARCH_MAX_CONCURRENCY" fi @@ -473,7 +485,12 @@ adaptive_print_sequence() { if (( concurrency == SEARCH_MAX_CONCURRENCY )); then break fi - local next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + local next_concurrency + if [[ -n "${SEARCH_ADDEND:-}" ]]; then + next_concurrency=$((concurrency + SEARCH_ADDEND)) + else + next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + fi if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then next_concurrency="$SEARCH_MAX_CONCURRENCY" fi From e0ae0715b14a29965fb9f339db6cd453b9e55037 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Tue, 14 Jul 2026 10:41:01 +0000 Subject: [PATCH 49/53] feat(adaptive): add RESUME_RUN_ID resume support and rename DSL to OSL - Add RESUME_RUN_ID env var for breakpoint resume in adaptive benchmarks. When set, the script reuses an existing result directory and skips already-tested (TP, DP, ISL, OSL) shapes based on adaptive_shapes.jsonl. - Fix RUN_ID unbound variable in resume mode. - Fix jq query to use -s (slurp) for jsonl files. - Add resume skip logic to DRY_RUN mode as well. - Rename DSL -> OSL across all adaptive benchmark files for consistency: - scripts/common/adaptive_bench_lib.sh - scripts/common/adaptive_concurrency.py - experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env - experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh - experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env - experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh - experiments/ADAPTIVE_CONCURRENCY_USAGE.md --- experiments/ADAPTIVE_CONCURRENCY_USAGE.md | 14 +- .../adaptive_config.env | 10 +- .../run_adaptive_concurrency.sh | 6 +- .../adaptive_config.env | 10 +- .../run_adaptive_concurrency.sh | 6 +- scripts/common/adaptive_bench_lib.sh | 124 ++++++++++++------ scripts/common/adaptive_concurrency.py | 22 ++-- 7 files changed, 121 insertions(+), 71 deletions(-) diff --git a/experiments/ADAPTIVE_CONCURRENCY_USAGE.md b/experiments/ADAPTIVE_CONCURRENCY_USAGE.md index 4e28428..3a1093b 100644 --- a/experiments/ADAPTIVE_CONCURRENCY_USAGE.md +++ b/experiments/ADAPTIVE_CONCURRENCY_USAGE.md @@ -1,6 +1,6 @@ # 自适应并发 Benchmark 使用说明 -这套脚本对每个固定的 `(TP, DP, ISL, DSL)` 依次测试 +这套脚本对每个固定的 `(TP, DP, ISL, OSL)` 依次测试 `C=1,2,4,8,...`，直到 Total TPS 连续两次增长不足 2%，或达到并发上限、 发生 OOM。每个正式测点发送 `C * 5` 个请求，并先用相同并发 C warm-up。 @@ -26,7 +26,7 @@ bash run_adaptive_concurrency.sh ## 2. 先 dry-run -只打印 TP/DP、ISL/DSL 和并发搜索计划，不加载模型： +只打印 TP/DP、ISL/OSL 和并发搜索计划，不加载模型： ```bash DRY_RUN=1 bash run_adaptive_concurrency.sh @@ -34,13 +34,13 @@ DRY_RUN=1 bash run_adaptive_concurrency.sh ## 3. 单组合 smoke -只测 `TP=8, DP=1, ISL=1K, DSL=128`，并发搜索到 8： +只测 `TP=8, DP=1, ISL=1K, OSL=128`，并发搜索到 8： ```bash RUN_ID=smoke-$(date +%Y%m%d-%H%M%S) \ TP_LIST="8" \ ISL_LIST="1024" \ -DSL_LIST="128" \ +OSL_LIST="128" \ GRID_LIMIT=1 \ SEARCH_MAX_CONCURRENCY=8 \ RESULT_BASE=/data3/yy/sskj_adaptive_results/smoke \ @@ -97,8 +97,8 @@ bash run_adaptive_concurrency.sh - vLLM 端口用 `VLLM_PORT`，SGLang 端口用 `SGLANG_PORT`。 - `TP_LIST="8"` 只跑 TP=8；DP 使用 `config.env` 中对应的 TP/DP 组合。 -- `ISL_LIST="1024 4096"`、`DSL_LIST="128 256"` 用来筛选长度。 -- `GRID_LIMIT=1` 表示每组 TP/DP 只跑一个 ISL/DSL shape。 +- `ISL_LIST="1024 4096"`、`OSL_LIST="128 256"` 用来筛选长度。 +- `GRID_LIMIT=1` 表示每组 TP/DP 只跑一个 ISL/OSL shape。 - `BENCH_WARMUP_MAX_REQUESTS=0` 表示每个测点用完整并发 C warm-up；正数表示上限。 - 正式基线保持 `BENCH_DATASET_NAME=random` 和 `RANDOM_RANGE_RATIO=1.0`。 - `random-ids` 只用于检查链路，不与正式 ShareGPT-backed 结果混用。 @@ -112,7 +112,7 @@ bash run_adaptive_concurrency.sh ```text adaptive_points.csv 每个并发测点的 TPS、TTFT、TPOT 等 -adaptive_summary.csv 每个 ISL/DSL/TP/DP 的饱和并发和最佳 TPS 并发 +adaptive_summary.csv 每个 ISL/OSL/TP/DP 的饱和并发和最佳 TPS 并发 adaptive_summary.md 可直接阅读的汇总表 run_manifest.json 本次模型、数据集和搜索参数 logs/ 编排与服务日志 diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env index 4085397..1eda1f0 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env @@ -1,6 +1,6 @@ # Adaptive concurrency search settings. # -# For each fixed (TP, DP, ISL, DSL), probe: +# For each fixed (TP, DP, ISL, OSL), probe: # C = start, start * multiplier, ... up to max # and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for # PLATEAU_PATIENCE consecutive points. @@ -21,7 +21,7 @@ PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" # random-ids to use generated token IDs without a ShareGPT seed dataset. BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed -# ISL/DSL points; lower values intentionally benchmark a length distribution. +# ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" # Before each measured point, warm up with the same concurrency so lazy kernel # compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no @@ -39,11 +39,11 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" # Optional space-separated filters, useful for smoke tests: -# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" TP_LIST="${TP_LIST:-}" ISL_LIST="${ISL_LIST:-}" -DSL_LIST="${DSL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" DRY_RUN="${DRY_RUN:-0}" -# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 90f7827..0979a46 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -130,7 +130,7 @@ engine_detect_oom() { engine_run_bench() { local isl="$1" - local dsl="$2" + local osl="$2" local concurrency="$3" local num_prompts="$4" local output_file="$5" @@ -142,7 +142,7 @@ engine_run_bench() { --port "$ENGINE_PORT" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" - --random-output-len "$dsl" + --random-output-len "$osl" --random-range-ratio "$RANDOM_RANGE_RATIO" --num-prompts "$num_prompts" --max-concurrency "$concurrency" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env index 4085397..1eda1f0 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env @@ -1,6 +1,6 @@ # Adaptive concurrency search settings. # -# For each fixed (TP, DP, ISL, DSL), probe: +# For each fixed (TP, DP, ISL, OSL), probe: # C = start, start * multiplier, ... up to max # and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for # PLATEAU_PATIENCE consecutive points. @@ -21,7 +21,7 @@ PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" # random-ids to use generated token IDs without a ShareGPT seed dataset. BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed -# ISL/DSL points; lower values intentionally benchmark a length distribution. +# ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" # Before each measured point, warm up with the same concurrency so lazy kernel # compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no @@ -39,11 +39,11 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" # Optional space-separated filters, useful for smoke tests: -# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" TP_LIST="${TP_LIST:-}" ISL_LIST="${ISL_LIST:-}" -DSL_LIST="${DSL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" DRY_RUN="${DRY_RUN:-0}" -# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index 8b1fdd0..ca23f6a 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/DSL shape. +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -111,7 +111,7 @@ engine_detect_oom() { engine_run_bench() { local isl="$1" - local dsl="$2" + local osl="$2" local concurrency="$3" local num_prompts="$4" local output_file="$5" @@ -123,7 +123,7 @@ engine_run_bench() { --port "$ENGINE_PORT" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" - --random-output-len "$dsl" + --random-output-len "$osl" --random-range-ratio "$RANDOM_RANGE_RATIO" --num-prompts "$num_prompts" --max-concurrency "$concurrency" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index c64f4f5..35dc5d2 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -115,7 +115,7 @@ adaptive_append_point_from_metrics() { local dp="$3" local mark="$4" local isl="$5" - local dsl="$6" + local osl="$6" local concurrency="$7" local num_prompts="$8" local attempt="$9" @@ -133,7 +133,7 @@ adaptive_append_point_from_metrics() { --argjson dp "$dp" \ --arg mark "$mark" \ --argjson isl "$isl" \ - --argjson dsl "$dsl" \ + --argjson osl "$osl" \ --argjson concurrency "$concurrency" \ --argjson num_prompts "$num_prompts" \ --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ @@ -149,7 +149,7 @@ adaptive_append_point_from_metrics() { dp: $dp, mark: $mark, isl: $isl, - dsl: $dsl, + osl: $osl, concurrency: $concurrency, num_prompts: $num_prompts, warmup_requests: $warmup_requests, @@ -167,7 +167,7 @@ adaptive_append_failed_point() { local dp="$2" local mark="$3" local isl="$4" - local dsl="$5" + local osl="$5" local concurrency="$6" local num_prompts="$7" local attempt="$8" @@ -183,7 +183,7 @@ adaptive_append_failed_point() { --argjson dp "$dp" \ --arg mark "$mark" \ --argjson isl "$isl" \ - --argjson dsl "$dsl" \ + --argjson osl "$osl" \ --argjson concurrency "$concurrency" \ --argjson num_prompts "$num_prompts" \ --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ @@ -199,7 +199,7 @@ adaptive_append_failed_point() { dp: $dp, mark: $mark, isl: $isl, - dsl: $dsl, + osl: $osl, concurrency: $concurrency, num_prompts: $num_prompts, warmup_requests: $warmup_requests, @@ -221,7 +221,7 @@ adaptive_append_shape_summary() { local dp="$2" local mark="$3" local isl="$4" - local dsl="$5" + local osl="$5" local status="$6" local stop_reason="$7" local tested_points="$8" @@ -245,7 +245,7 @@ adaptive_append_shape_summary() { --argjson dp "$dp" \ --arg mark "$mark" \ --argjson isl "$isl" \ - --argjson dsl "$dsl" \ + --argjson osl "$osl" \ --arg status "$status" \ --arg stop_reason "$stop_reason" \ --argjson tested_points "$tested_points" \ @@ -263,7 +263,7 @@ adaptive_append_shape_summary() { dp: $dp, mark: $mark, isl: $isl, - dsl: $dsl, + osl: $osl, status: $status, stop_reason: $stop_reason, tested_points: $tested_points, @@ -281,7 +281,7 @@ adaptive_run_point() { local tp="$1" local dp="$2" local isl="$3" - local dsl="$4" + local osl="$4" local concurrency="$5" local num_prompts="$6" local config_root="$7" @@ -295,17 +295,17 @@ adaptive_run_point() { for (( attempt = 1; attempt <= MAX_POINT_RETRIES + 1; attempt++ )); do POINT_ATTEMPT="$attempt" - POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${dsl}_a${attempt}.jsonl" - POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${dsl}_a${attempt}.log" - POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${dsl}_a${attempt}.json" - local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${dsl}_a${attempt}.csv" + POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${osl}_a${attempt}.jsonl" + POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.log" + POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.json" + local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${osl}_a${attempt}.csv" - log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}" + log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} osl=${osl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}" gpu_pid="$(adaptive_start_gpu_monitor "$gpu_csv")" bench_rc=0 timeout "$SCENARIO_TIMEOUT_S" bash -c \ 'engine_run_bench "$@"' _ \ - "$isl" "$dsl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \ + "$isl" "$osl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \ > "$POINT_DETAIL_LOG" 2>&1 || bench_rc=$? adaptive_stop_gpu_monitor "$gpu_pid" @@ -316,7 +316,7 @@ adaptive_run_point() { --output "$POINT_METRICS_FILE" \ --expected-prompts "$num_prompts" \ --isl "$isl" \ - --dsl "$dsl" \ + --osl "$osl" \ --input-tolerance-pct "$INPUT_LENGTH_TOLERANCE_PCT" \ --output-tolerance-pct "$OUTPUT_LENGTH_TOLERANCE_PCT" \ >> "$POINT_DETAIL_LOG" 2>&1 || parse_rc=$? @@ -350,7 +350,7 @@ adaptive_run_shape() { local dp="$2" local mark="$3" local isl="$4" - local dsl="$5" + local osl="$5" local config_root="$6" local concurrency="$SEARCH_START_CONCURRENCY" local previous_tps="" @@ -376,7 +376,7 @@ adaptive_run_shape() { while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER)) point_rc=0 - adaptive_run_point "$tp" "$dp" "$isl" "$dsl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$? + adaptive_run_point "$tp" "$dp" "$isl" "$osl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$? tested_points=$((tested_points + 1)) stop_probe_concurrency="$concurrency" @@ -409,7 +409,7 @@ adaptive_run_shape() { max_successful_concurrency="$concurrency" last_total_tps="$current_tps" adaptive_append_point_from_metrics \ - "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$dsl" \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" @@ -422,7 +422,7 @@ adaptive_run_shape() { previous_tps="$current_tps" elif (( point_rc == 20 )); then adaptive_append_failed_point \ - "$tp" "$dp" "$mark" "$isl" "$dsl" "$concurrency" "$num_prompts" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ "$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" shape_status="OOM_BOUNDARY" stop_reason="OOM" @@ -430,7 +430,7 @@ adaptive_run_shape() { break elif (( point_rc == 21 )); then adaptive_append_point_from_metrics \ - "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$dsl" \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "" 0 \ "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" shape_status="INVALID_WORKLOAD" @@ -439,7 +439,7 @@ adaptive_run_shape() { break else adaptive_append_failed_point \ - "$tp" "$dp" "$mark" "$isl" "$dsl" "$concurrency" "$num_prompts" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ "$POINT_ATTEMPT" "FAILED" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" shape_status="FAILED" stop_reason="BENCHMARK_FAILED" @@ -464,11 +464,11 @@ adaptive_run_shape() { done adaptive_append_shape_summary \ - "$tp" "$dp" "$mark" "$isl" "$dsl" "$shape_status" "$stop_reason" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$shape_status" "$stop_reason" \ "$tested_points" "$max_successful_concurrency" "$saturation_concurrency" \ "$stop_probe_concurrency" "$best_concurrency" "$best_tps" "$last_total_tps" - log "shape done tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}" + log "shape done tp=${tp} dp=${dp} isl=${isl} osl=${osl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}" } adaptive_matches_filter() { @@ -507,19 +507,61 @@ adaptive_cleanup_active_server() { fi } +adaptive_shape_already_tested() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local osl="$5" + local resume_shapes_jsonl="$6" + if [[ ! -f "$resume_shapes_jsonl" ]]; then + return 1 + fi + local count + count="$(jq -s -r --argjson tp "$tp" --argjson dp "$dp" --arg mark "$mark" --argjson isl "$isl" --argjson osl "$osl" ' + [ .[] | select(.tp == $tp and .dp == $dp and .mark == $mark and .isl == $isl and .osl == $osl) ] | length + ' "$resume_shapes_jsonl" 2>/dev/null)" + if [[ -z "$count" ]]; then + return 1 + fi + if (( count > 0 )); then + return 0 + fi + return 1 +} + adaptive_main() { - RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}" + local resume_mode=0 + local resume_shapes_jsonl="" + if [[ -n "${RESUME_RUN_ID:-}" ]]; then + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RESUME_RUN_ID}" + if [[ ! -d "$ADAPTIVE_RUN_ROOT" ]]; then + log "ERROR: RESUME_RUN_ID=${RESUME_RUN_ID} does not exist under ${RESULT_BASE}" + return 1 + fi + resume_mode=1 + RUN_ID="$RESUME_RUN_ID" + resume_shapes_jsonl="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" + if [[ ! -f "$resume_shapes_jsonl" ]]; then + log "WARNING: no adaptive_shapes.jsonl found in ${ADAPTIVE_RUN_ROOT}; will resume with empty history" + fi + log "RESUME mode: continuing run_id=${RESUME_RUN_ID}" + else + RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}" + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}" + fi RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" - ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}" ADAPTIVE_LOG_DIR="${ADAPTIVE_RUN_ROOT}/logs" ADAPTIVE_POINTS_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_points.jsonl" ADAPTIVE_SHAPES_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" local shapes_tsv="${ADAPTIVE_RUN_ROOT}/shapes.tsv" mkdir -p "$ADAPTIVE_LOG_DIR" - : > "$ADAPTIVE_POINTS_JSONL" - : > "$ADAPTIVE_SHAPES_JSONL" + if (( resume_mode == 0 )); then + : > "$ADAPTIVE_POINTS_JSONL" + : > "$ADAPTIVE_SHAPES_JSONL" + fi log_init "${ADAPTIVE_LOG_DIR}/orchestrator.log" adaptive_validate_config @@ -611,7 +653,7 @@ adaptive_main() { trap 'adaptive_cleanup_active_server; exit 143' TERM local abort_all=0 - local cfg tp dp config_root server_args shape_count mark isl dsl + local cfg tp dp config_root server_args shape_count mark isl osl for cfg in "${PARALLEL_CONFIGS[@]}"; do read -r tp dp <<< "$cfg" if ! adaptive_matches_filter "$tp" "$TP_LIST"; then @@ -626,15 +668,19 @@ adaptive_main() { if [[ "$DRY_RUN" == "1" ]]; then log "DRY_RUN server tp=${tp} dp=${dp}: ${server_args}" shape_count=0 - while IFS=$'\t' read -r mark isl dsl; do + while IFS=$'\t' read -r mark isl osl; do [[ "$mark" == "mark" ]] && continue adaptive_matches_filter "$isl" "$ISL_LIST" || continue - adaptive_matches_filter "$dsl" "$DSL_LIST" || continue + adaptive_matches_filter "$osl" "$OSL_LIST" || continue if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then break fi shape_count=$((shape_count + 1)) - log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} C=[$(adaptive_print_sequence)]" + if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then + log "DRY_RUN RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}" + continue + fi + log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} osl=${osl} C=[$(adaptive_print_sequence)]" done < "$shapes_tsv" continue fi @@ -653,17 +699,21 @@ adaptive_main() { } shape_count=0 - while IFS=$'\t' read -r mark isl dsl; do + while IFS=$'\t' read -r mark isl osl; do [[ "$mark" == "mark" ]] && continue adaptive_matches_filter "$isl" "$ISL_LIST" || continue - adaptive_matches_filter "$dsl" "$DSL_LIST" || continue + adaptive_matches_filter "$osl" "$OSL_LIST" || continue if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then log "GRID_LIMIT=${GRID_LIMIT} reached for tp=${tp} dp=${dp}" break fi shape_count=$((shape_count + 1)) - log "===== shape START tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} mark=${mark} =====" - adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$dsl" "$config_root" + if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then + log "RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}" + continue + fi + log "===== shape START tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark} =====" + adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$osl" "$config_root" if (( ADAPTIVE_FATAL_WORKLOAD == 1 )); then log "ERROR: invalid synthetic workload; aborting remaining searches" diff --git a/scripts/common/adaptive_concurrency.py b/scripts/common/adaptive_concurrency.py index d7a517b..e8de21a 100755 --- a/scripts/common/adaptive_concurrency.py +++ b/scripts/common/adaptive_concurrency.py @@ -17,7 +17,7 @@ POINT_FIELDS = [ "dp", "mark", "isl", - "dsl", + "osl", "concurrency", "num_prompts", "warmup_requests", @@ -60,7 +60,7 @@ SHAPE_FIELDS = [ "dp", "mark", "isl", - "dsl", + "osl", "status", "stop_reason", "tested_points", @@ -114,16 +114,16 @@ def command_shapes(args: argparse.Namespace) -> int: mode = args.mode or data.get("mode", "Y") print("mark\tinput_len\toutput_len") for isl_text in sorted(data["matrix"], key=int): - dsl_map = data["matrix"][isl_text] - for dsl_text in sorted(dsl_map, key=int): - mark = dsl_map[dsl_text] + osl_map = data["matrix"][isl_text] + for osl_text in sorted(osl_map, key=int): + mark = osl_map[osl_text] if mode == "Y" and mark != "Y": continue if mode == "Y+P" and mark not in ("Y", "P"): continue if mode != "all" and mark == "N": continue - print(f"{mark}\t{int(isl_text)}\t{int(dsl_text)}") + print(f"{mark}\t{int(isl_text)}\t{int(osl_text)}") return 0 @@ -175,8 +175,8 @@ def command_parse_result(args: argparse.Namespace) -> int: f"mean_input_tokens={mean_input:.2f}, expected_range={input_low:.2f}-{input_high:.2f}" ) - output_low = args.dsl * (1.0 - args.output_tolerance_pct / 100.0) - output_high = args.dsl * (1.0 + args.output_tolerance_pct / 100.0) + output_low = args.osl * (1.0 - args.output_tolerance_pct / 100.0) + output_high = args.osl * (1.0 + args.output_tolerance_pct / 100.0) if not output_low <= mean_output <= output_high: validation_errors.append( f"mean_output_tokens={mean_output:.2f}, expected_range={output_low:.2f}-{output_high:.2f}" @@ -256,12 +256,12 @@ def command_summarize(args: argparse.Namespace) -> int: with report.open("w", encoding="utf-8") as f: f.write("# Adaptive concurrency search summary\n\n") f.write( - "| Engine | TP | DP | ISL | DSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C |\n" + "| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C |\n" ) f.write("|---|---:|---:|---:|---:|---|---:|---:|---:|---:|\n") for row in shapes: f.write( - "| {engine} | {tp} | {dp} | {isl} | {dsl} | {stop_reason} | " + "| {engine} | {tp} | {dp} | {isl} | {osl} | {stop_reason} | " "{saturation_concurrency} | {best_tps_concurrency} | {best_total_tps} | " "{max_successful_concurrency} |\n".format(**{k: row.get(k, "") for k in SHAPE_FIELDS}) ) @@ -292,7 +292,7 @@ def build_parser() -> argparse.ArgumentParser: parse_result.add_argument("--output", type=Path, required=True) parse_result.add_argument("--expected-prompts", type=int, required=True) parse_result.add_argument("--isl", type=int, required=True) - parse_result.add_argument("--dsl", type=int, required=True) + parse_result.add_argument("--osl", type=int, required=True) parse_result.add_argument("--input-tolerance-pct", type=float, default=5.0) parse_result.add_argument("--output-tolerance-pct", type=float, default=10.0) parse_result.set_defaults(func=command_parse_result) From 5405d002fcdd952731e52941308e3d5cc2099b12 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Tue, 14 Jul 2026 10:47:09 +0000 Subject: [PATCH 50/53] chore: remove legacy benchmark files and scripts Delete obsolete files that are no longer maintained: - experiments/legacy_bench_results/ (old benchmark reports and summaries) - scripts/benchmark_dspark_0707/ (outdated dspark benchmark scripts) - scripts/start_dsv4_dspark_8card.sh (deprecated deployment script) --- experiments/legacy_bench_results/README.md | 25 -- .../dspark_grid_20260707-132641/evaluation.md | 167 ----------- .../dspark_grid_20260707-132641/report.md | 120 -------- .../comparison_report.md | 129 --------- .../README.md | 102 ------- .../summary.json | 197 ------------- .../eagle_grid/dspark_vs_eagle_report.md | 96 ------- .../legacy_bench_results/eagle_grid/report.md | 6 - .../summary.json | 58 ---- .../length_distribution.png | Bin 150830 -> 0 bytes .../report.md | 267 ------------------ .../summary.json | 156 ---------- .../summary.json | 39 --- .../summary.json | 59 ---- .../summary.json | 131 --------- scripts/benchmark_dspark_0707/README.md | 60 ---- .../bench_dspark_focused.sh | 64 ----- .../benchmark_dspark_0707/bench_dspark_p1.sh | 60 ---- .../benchmark_dspark_0707/bench_dspark_p2.sh | 63 ----- .../benchmark_dspark_0707/bench_dspark_p3.sh | 61 ---- .../parse_eagle_vs_dspark.py | 258 ----------------- .../benchmark_dspark_0707/parse_results.py | 120 -------- .../parse_st_comparison.py | 174 ------------ .../run_dspark_benchmark_grid.sh | 114 -------- .../run_dspark_st_comparison.sh | 103 ------- scripts/start_dsv4_dspark_8card.sh | 65 ----- 26 files changed, 2694 deletions(-) delete mode 100644 experiments/legacy_bench_results/README.md delete mode 100644 experiments/legacy_bench_results/dspark_grid_20260707-132641/evaluation.md delete mode 100644 experiments/legacy_bench_results/dspark_grid_20260707-132641/report.md delete mode 100644 experiments/legacy_bench_results/dspark_st_comparison_20260707-150649/comparison_report.md delete mode 100644 experiments/legacy_bench_results/dsv4_backend_comparison_20260707/README.md delete mode 100644 experiments/legacy_bench_results/dsv4_comparison_20260705_152221/summary.json delete mode 100644 experiments/legacy_bench_results/eagle_grid/dspark_vs_eagle_report.md delete mode 100644 experiments/legacy_bench_results/eagle_grid/report.md delete mode 100644 experiments/legacy_bench_results/sglang_8card_max_throughput_20260705_030839/summary.json delete mode 100644 experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/length_distribution.png delete mode 100644 experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/report.md delete mode 100644 experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/summary.json delete mode 100644 experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121046/summary.json delete mode 100644 experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121218/summary.json delete mode 100644 experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121256/summary.json delete mode 100644 scripts/benchmark_dspark_0707/README.md delete mode 100755 scripts/benchmark_dspark_0707/bench_dspark_focused.sh delete mode 100755 scripts/benchmark_dspark_0707/bench_dspark_p1.sh delete mode 100755 scripts/benchmark_dspark_0707/bench_dspark_p2.sh delete mode 100755 scripts/benchmark_dspark_0707/bench_dspark_p3.sh delete mode 100644 scripts/benchmark_dspark_0707/parse_eagle_vs_dspark.py delete mode 100644 scripts/benchmark_dspark_0707/parse_results.py delete mode 100644 scripts/benchmark_dspark_0707/parse_st_comparison.py delete mode 100755 scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh delete mode 100755 scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh delete mode 100755 scripts/start_dsv4_dspark_8card.sh diff --git a/experiments/legacy_bench_results/README.md b/experiments/legacy_bench_results/README.md deleted file mode 100644 index 6a9698c..0000000 --- a/experiments/legacy_bench_results/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Legacy Bench Results Archive - -This directory contains historical benchmark results that were originally stored under `bench_results/` before the `experiments/` layout was introduced. - -All legacy results have been moved here; `bench_results/` itself no longer exists. - -## Contents - -| Directory | Original location | Description | -|---|---|---| -| `dspark_grid_20260707-132641/` | `bench_results/dspark_grid_20260707-132641/` | P1/P2/P3 DSpark grid benchmark | -| `dspark_st_comparison_20260707-150649/` | `bench_results/dspark_st_comparison_20260707-150649/` | `--spec-tokens 3` vs `5` comparison | -| `dsv4_backend_comparison_20260707/` | `bench_results/dsv4_backend_comparison_20260707/` | SGLang vs vLLM backend comparison raw outputs manifest | -| `dsv4_comparison_20260705_152221/` | `bench_results/dsv4_comparison_20260705_152221/` | Early vLLM-dspark configuration comparison | -| `dsv4_flash_dspark_misc/` | `bench_results/dsv4_flash_dspark_misc/` | Early DSV4 DSpark one-off tests | -| `eagle_grid/` | `bench_results/eagle_grid/` | SGLang EAGLE vs DSpark comparison | -| `pd_bench/` | `bench_results/pd_bench/` | PD disaggregation benchmark outputs | -| `sglang_8card_max_throughput_20260705_030839/` | `bench_results/sglang_8card_max_throughput_20260705_030839/` | SGLang 8-card max throughput scan | -| `sglang_8card_systematic_20260704_120819/` | `bench_results/sglang_8card_systematic_20260704_120819/` | Early SGLang systematic scan | -| `sglang_misc/` | `bench_results/sglang_misc/` | Miscellaneous SGLang outputs | -| `vllm_dspark_qwen3_20260705_121046/` | `bench_results/vllm_dspark_qwen3_20260705_121046/` | Qwen3 DSpark tests | -| `vllm_dspark_qwen3_20260705_121218/` | `bench_results/vllm_dspark_qwen3_20260705_121218/` | Qwen3 DSpark tests | -| `vllm_dspark_qwen3_20260705_121256/` | `bench_results/vllm_dspark_qwen3_20260705_121256/` | Qwen3 DSpark tests | - -> Note: Some internal file paths (e.g. `result_file` in `summary.json`, hardcoded paths in old scripts) still reference the original `bench_results/` locations. These are kept as-is for historical accuracy. diff --git a/experiments/legacy_bench_results/dspark_grid_20260707-132641/evaluation.md b/experiments/legacy_bench_results/dspark_grid_20260707-132641/evaluation.md deleted file mode 100644 index d0663c9..0000000 --- a/experiments/legacy_bench_results/dspark_grid_20260707-132641/evaluation.md +++ /dev/null @@ -1,167 +0,0 @@ -# vllm-dspark Benchmark 结果评估 - -> 对应结果：`/data/user1/yy/experiments/legacy_bench_results/dspark_grid_20260707-132641` -> 运行时间：2026-07-07 -> 硬件：8× NVIDIA H200 143GB -> 模型：`/data/models/DeepSeek-V4-Flash-DSpark` -> 服务配置：TP=8，FP8 KV cache，`--spec-method dspark --spec-tokens 5`，block-size=256，max-num-seqs=256 -> 压测工具：`sglang.bench_serving --backend vllm` - ---- - -## 1. 总体结论 - -本次 grid 测试覆盖了从轻量 chat 到超长上下文、从重 decode 到高并发压测的多种场景。整体上看： - -- **DSpark 在短输入、中高并发场景下表现优秀**，`chat_short` 在并发 64 时达到约 9580 tok/s，`stress_standard` 在并发 128 时达到约 17465 tok/s。 -- **低并发下 DSpark 的 draft 开销明显**，部分场景单并发延迟和吞吐都不如预期。 -- **超长上下文场景存在明显的吞吐拐点**，`long_rag`（32K 输入）在并发 4 达到峰值后，并发 8 吞吐腰斩。 -- **延迟长尾（P95/P99）普遍较重**，提示调度、内存或投机解码验证阶段存在抖动。 - ---- - -## 2. 异常点分析 - -### 2.1 低并发（c=1）下 DSpark 收益被开销吃掉 - -以 `chat_standard`（input=1000, output=256）为例： - -| 并发 | req/s | mean E2E(ms) | P99 E2E(ms) | mean TTFT(ms) | P99 TTFT(ms) | -|---|---:|---:|---:|---:|---:| -| 1 | 0.49 | 2040.65 | 10715.53 | 1412.48 | 8430.71 | -| 8 | 4.35 | 1819.35 | 15165.63 | 327.62 | 4404.92 | -| 16 | 8.32 | 1885.15 | 10491.31 | 443.50 | 8567.07 | -| 32 | 10.00 | 3156.73 | 12936.08 | 560.77 | 7129.66 | - -- **单并发吞吐仅 0.49 req/s**，远低于无投机基线可预期的水平。 -- **P99 TTFT 高达 8.4s**，说明单请求首次预填充极不稳定，可能是 draft 验证/编译缓存未命中或 warmup 不充分导致。 -- 并发提升到 8 后，TTFT 均值下降到 327ms，说明 DSpark 的 draft 计算需要 batch 才能摊薄。 - -**判断**：这是 DSpark 的典型特征——低并发下 draft 固定开销无法被摊薄，导致延迟尾和单请求吞吐都较差。 - -### 2.2 超长上下文出现吞吐断崖 - -`long_rag`（input=32000, output=512）： - -| 并发 | Total tok/s | mean TTFT(ms) | P99 TTFT(ms) | -|---|---:|---:|---:| -| 1 | 16104.98 | 549.72 | 1163.96 | -| 2 | 43215.62 | 192.57 | 409.32 | -| 4 | **70040.54** | 191.57 | 427.69 | -| 8 | 34141.70（↓51%） | 930.98 | 2742.48 | - -`long_context_probe`（input=16000, output=512）也有类似模式： - -| 并发 | Total tok/s | mean TTFT(ms) | -|---|---:|---:| -| 1 | 11051.09 | 316.59 | -| 2 | 26862.06 | 110.65 | -| 4 | 43042.77 | 127.38 | -| 8 | 29255.46（↓32%） | 453.93 | -| 16 | 43292.68 | 444.14 | - -**判断**： -- 32K/16K 长 prefill 在并发 4 时达到最佳，继续加并发反而下降，可能与 **KV cache 显存带宽瓶颈**、**prefill 阶段 scheduling 阻塞** 或 **hybrid KV cache manager 的换入换出** 有关。 -- `long_rag` 在 c=8 时 P99 TTFT 暴涨到 2.7s，进一步印证内存/调度压力。 - -### 2.3 延迟长尾普遍偏重 - -几乎所有场景的 P99 E2E 都是 mean E2E 的 2~4 倍： - -| 场景 | 并发 | mean E2E | P99 E2E | 倍数 | -|---|---:|---:|---:|---:| -| chat_standard | 32 | 3156.73 | 12936.08 | 4.1× | -| generation_standard | 32 | 2908.89 | 6938.03 | 2.4× | -| summarization | 32 | 5587.21 | 14541.75 | 2.6× | -| decode_heavy | 32 | 5562.57 | 12822.35 | 2.3× | -| stress_standard | 128 | 4417.43 | 14850.43 | 3.4× | - -**判断**：投机解码的 draft 验证失败会导致 fall back 到逐个 target 前向，造成明显的延迟毛刺；另外 vLLM v1 引擎的 scheduling 在高并发下也可能产生队列等待。 - -### 2.4 高并发下 TPOT/ITL 增长较快 - -`stress_standard`（input=1000, output=256）： - -| 并发 | mean TPOT | P95 TPOT | P99 TPOT | mean ITL | -|---|---:|---:|---:|---:| -| 96 | 29.46 | 67.30 | 123.42 | 113.64 | -| 128 | 35.20 | 76.85 | 141.17 | 136.57 | - -TPOT 从 29ms 升到 35ms，P99 超过 140ms，说明 decode 阶段在极限并发下已经接近饱和。 - ---- - -## 3. 可重点优化的方向 - -### 3.1 按并发动态调整 `--spec-tokens` - -参考历史对比数据（`dsv4_inference_comparison_report.md`）： - -| spec-tokens | 低并发（c=1） | 高并发（c=64） | -|---|---|---| -| 3 | 1.03 req/s, 254 tok/s | **14.53 req/s, 3507 tok/s** | -| 5 | **1.07 req/s, 257 tok/s** | 9.36 req/s, 2210 tok/s | -| 7 | 1.03 req/s, 251 tok/s | 9.13 req/s, 2218 tok/s | - -当前部署固定使用 `--spec-tokens 5`： -- 对低并发（c=1~16）相对友好； -- 对高并发（c≥64）不是最优，st=3 在高并发下接受率更高、验证开销更小。 - -**建议**： -- 高并发服务场景：改用 `--spec-tokens 3`。 -- 若负载混合，可考虑不同 endpoint/队列按并发分桶，或测试 st=3/st=5 在本 grid 下的完整表现。 - -### 3.2 长上下文场景优化 - -针对 `long_rag`（32K）和 `long_context_probe`（16K）的吞吐断崖： - -1. **KV cache dtype 对比测试**：当前是 FP8，可测试 BF16 是否改善长上下文稳定性和吞吐。 -2. **block-size 调整**：当前 256，可尝试 128 或 64 看是否减少内存碎片。 -3. **hybrid KV cache manager**：当前 `--no-disable-hybrid-kv-cache-manager`，可对比关闭后的表现。 -4. **限制长上下文并发**：若业务允许，对 32K 输入设置更低的 `max-num-seqs` 或独立队列，避免 c=8 后的内存带宽瓶颈。 -5. **attention backend**：可测试 `FLASHINFER_MLA_SPARSE_DSV4`（脚本 `start_dsv4_dspark_8card_flashinfer.sh`）在长上下文下的表现。 - -### 3.3 降低延迟长尾 - -1. **增加 warmup**：当前仅 10 条 warmup。DSpark 的 JIT 编译/算子 warmup 在首次运行时较重，建议增加到 50~100 条或先跑一轮 discard。 -2. **compilation cache 持久化**：确认 `TORCH_EXTENSIONS_DIR`、`VLLM_CACHE_ROOT` 等环境变量已设置并复用，避免每次重启重新编译 deep_gemm/tilelang。 -3. **调度参数**：尝试调整 `--max-num-batched-tokens`、`--max-num-seqs`、chunked prefill 相关参数，减少队列等待。 -4. **P99 TTFT 抖动**：高并发下 P99 TTFT 高通常与 prefill batching 有关，可尝试限制 prefill 阶段 batch 大小或启用 prompt chunked prefill。 - -### 3.4 低并发场景是否需要 DSpark - -对于 `chat_standard` c=1： -- 当前 0.49 req/s、P99 E2E 10.7s 的服务质量较差。 -- 若业务存在大量单用户/低并发请求，可考虑： - - 部署无投机基线 vLLM 专门服务低并发流量； - - 或设置最小 batch 阈值，累积少量请求后再用 DSpark 处理。 - -### 3.5 对比 baseline 与 SGLang - -本次测试只有 DSpark 单一配置，建议补充： -1. **vLLM-dspark 无投机基线**：用同样的 `sglang.bench_serving --backend vllm` 测一遍相同 grid，计算真实加速比。 -2. **SGLang EAGLE**：在相同 grid 下复测，验证 DSpark 在不同场景下的相对优势。 -3. **不同 `--spec-tokens` 的完整 grid**：st=3 和 st=5 各跑一遍，绘制并发-吞吐曲线。 - ---- - -## 4. 推荐下一步实验 - -| 优先级 | 实验 | 预期收益 | -|---|---|---| -| P0 | 高并发改用 `--spec-tokens 3` | 提升 c≥64 场景吞吐 20~50% | -| P0 | 长上下文（16K/32K）对比 BF16 KV cache | 改善吞吐断崖和 P99 TTFT | -| P1 | 增加 warmup / 预热缓存 | 降低 P99 TTFT 和首请求延迟 | -| P1 | 测试 `FLASHINFER_MLA_SPARSE_DSV4` backend | 可能提升长上下文吞吐 | -| P2 | 跑无投机基线 grid | 获得真实加速比 | -| P2 | 对比 st=3/st=5 完整曲线 | 找到最佳投机长度配置 | - ---- - -## 5. 结论 - -当前 DSpark 部署在 **短输入、中高并发** 场景下已经展现出明显优势，但在 **低并发** 和 **超长上下文** 场景下存在明显短板。最优先的优化是： - -1. 高并发服务改用 `--spec-tokens 3`； -2. 针对 16K/32K 长上下文做 KV cache dtype 和 attention backend 的调优； -3. 补充 baseline 测试，量化 DSpark 的真实收益。 diff --git a/experiments/legacy_bench_results/dspark_grid_20260707-132641/report.md b/experiments/legacy_bench_results/dspark_grid_20260707-132641/report.md deleted file mode 100644 index 6b0ce76..0000000 --- a/experiments/legacy_bench_results/dspark_grid_20260707-132641/report.md +++ /dev/null @@ -1,120 +0,0 @@ -# vllm-dspark Benchmark Grid Report - -- Result root: `/data/user1/yy/experiments/legacy_bench_results/dspark_grid_20260707-132641` -- Model: `/data/models/DeepSeek-V4-Flash-DSpark` -- Backend: vllm-dspark (TP=8, FP8 KV cache, spec-method=dspark, spec-tokens=5) -- Benchmark client: `sglang.bench_serving --backend vllm` - -## p1_quick - -### chat_standard (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 65.33 | 32 | 0.49 | 195.58 | 71.28 | 266.86 | 2040.65 | 8241.20 | 10715.53 | 1412.48 | 6903.30 | 8430.71 | 6.41 | 32.72 | 51.29 | 16.61 | -| 8 | 29.42 | 128 | 4.35 | 2130.85 | 569.52 | 2700.37 | 1819.35 | 8891.79 | 15165.63 | 327.62 | 379.39 | 4404.92 | 11.58 | 51.06 | 82.44 | 51.00 | -| 16 | 30.76 | 256 | 8.32 | 4144.89 | 1144.74 | 5289.63 | 1885.15 | 8902.10 | 10491.31 | 443.50 | 2779.49 | 8567.07 | 12.10 | 23.11 | 83.70 | 46.18 | -| 32 | 51.18 | 512 | 10.00 | 5017.88 | 1331.65 | 6349.53 | 3156.73 | 10438.92 | 12936.08 | 560.77 | 5750.71 | 7129.66 | 22.09 | 66.23 | 171.57 | 85.81 | - -### generation_standard (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 36.24 | 32 | 0.88 | 352.60 | 479.25 | 831.85 | 1131.63 | 1979.98 | 2576.42 | 66.57 | 141.80 | 177.56 | 1.94 | 2.85 | 2.88 | 8.50 | -| 8 | 27.43 | 128 | 4.67 | 2285.21 | 2362.35 | 4647.56 | 1649.16 | 3162.33 | 3844.70 | 80.88 | 157.80 | 160.50 | 3.23 | 5.31 | 7.09 | 14.77 | -| 16 | 35.39 | 256 | 7.23 | 3602.52 | 3654.79 | 7257.31 | 2133.90 | 4187.67 | 5500.20 | 88.78 | 169.39 | 199.78 | 4.37 | 6.97 | 11.81 | 19.28 | -| 32 | 48.16 | 512 | 10.63 | 5333.41 | 5371.85 | 10705.25 | 2908.89 | 5849.64 | 6938.03 | 112.84 | 211.64 | 405.78 | 5.84 | 9.30 | 12.47 | 25.89 | - -### summarization (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 33.63 | 32 | 0.95 | 4247.40 | 516.49 | 4763.90 | 1049.88 | 1783.33 | 1848.92 | 178.56 | 292.66 | 314.18 | 1.60 | 1.78 | 2.41 | 8.62 | -| 4 | 10.99 | 32 | 2.91 | 13000.06 | 1580.84 | 14580.90 | 1301.15 | 2238.30 | 2324.70 | 101.68 | 187.78 | 188.57 | 2.18 | 2.47 | 3.49 | 11.86 | -| 8 | 33.60 | 128 | 3.81 | 15369.23 | 1908.96 | 17278.19 | 2072.22 | 3978.11 | 4531.26 | 214.01 | 502.25 | 630.15 | 3.70 | 5.52 | 7.98 | 19.60 | -| 16 | 46.93 | 256 | 5.45 | 22698.72 | 2761.38 | 25460.10 | 2840.57 | 6243.39 | 7179.11 | 235.75 | 552.81 | 740.45 | 5.45 | 9.62 | 12.75 | 27.77 | - -## p2_core - -### chat_short (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 11.39 | 32 | 2.81 | 717.30 | 408.87 | 1126.18 | 355.00 | 685.00 | 815.55 | 48.57 | 105.64 | 105.97 | 2.07 | 3.10 | 3.69 | 8.51 | -| 8 | 10.62 | 128 | 12.06 | 3320.19 | 1591.90 | 4912.09 | 647.05 | 1401.33 | 1648.15 | 78.91 | 151.78 | 155.67 | 4.59 | 7.20 | 12.54 | 17.85 | -| 16 | 14.89 | 256 | 17.19 | 4533.71 | 2394.66 | 6928.36 | 902.72 | 1879.32 | 2211.23 | 95.95 | 165.09 | 217.14 | 6.19 | 10.26 | 15.12 | 23.65 | -| 32 | 24.90 | 512 | 20.56 | 5458.26 | 2723.85 | 8182.10 | 1508.89 | 3076.09 | 3881.52 | 161.04 | 292.00 | 383.41 | 10.80 | 18.19 | 25.45 | 41.86 | -| 64 | 21.27 | 512 | 24.07 | 6391.14 | 3189.39 | 9580.52 | 2547.81 | 5606.07 | 6543.97 | 268.68 | 543.21 | 588.63 | 18.24 | 30.39 | 39.57 | 70.29 | - -### chat_standard (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 64 | 19.87 | 512 | 25.76 | 12922.80 | 3429.47 | 16352.28 | 2380.94 | 4838.43 | 6315.98 | 291.17 | 794.65 | 947.26 | 16.87 | 30.96 | 42.55 | 68.46 | - -### generation_standard (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 64 | 43.59 | 512 | 11.75 | 5892.47 | 5934.93 | 11827.40 | 5136.14 | 10244.70 | 12575.20 | 209.70 | 536.96 | 606.58 | 10.33 | 17.41 | 21.20 | 45.47 | - -### long_context_probe (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 23.79 | 32 | 1.34 | 10684.78 | 366.30 | 11051.09 | 742.62 | 1237.28 | 1418.82 | 316.59 | 604.23 | 617.85 | 1.54 | 1.72 | 1.85 | 8.53 | -| 2 | 9.79 | 32 | 3.27 | 25971.67 | 890.38 | 26862.06 | 604.82 | 1055.84 | 1116.41 | 110.65 | 194.62 | 206.64 | 1.77 | 2.04 | 2.13 | 9.90 | -| 4 | 6.11 | 32 | 5.24 | 41616.05 | 1426.72 | 43042.77 | 741.94 | 1262.13 | 1447.68 | 127.38 | 222.70 | 235.35 | 2.26 | 2.69 | 3.69 | 12.39 | -| 8 | 36.07 | 128 | 3.55 | 28357.82 | 897.65 | 29255.46 | 2232.38 | 4946.97 | 5371.22 | 453.93 | 1089.99 | 1182.47 | 7.06 | 13.03 | 20.03 | 38.51 | -| 16 | 49.08 | 256 | 5.22 | 41910.25 | 1382.43 | 43292.68 | 3028.26 | 7651.28 | 9531.10 | 444.14 | 1212.92 | 1368.04 | 10.00 | 20.71 | 30.74 | 53.07 | - -### rag_medium (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 18.21 | 32 | 1.76 | 3568.86 | 494.63 | 4063.49 | 568.24 | 966.60 | 1201.99 | 110.94 | 151.11 | 154.78 | 1.63 | 1.95 | 2.50 | 8.60 | -| 4 | 6.31 | 32 | 5.07 | 10300.85 | 1427.65 | 11728.51 | 751.19 | 1259.07 | 1593.86 | 92.09 | 161.03 | 187.47 | 2.34 | 3.25 | 4.02 | 12.33 | -| 8 | 19.13 | 128 | 6.69 | 14147.80 | 1692.32 | 15840.12 | 1166.59 | 2199.67 | 2982.23 | 148.09 | 253.10 | 350.67 | 4.30 | 6.78 | 9.92 | 21.06 | -| 16 | 25.89 | 256 | 9.89 | 20561.53 | 2622.36 | 23183.89 | 1587.09 | 3316.74 | 4019.02 | 164.29 | 303.79 | 389.34 | 5.50 | 8.80 | 14.38 | 27.81 | -| 32 | 39.47 | 512 | 12.97 | 26523.95 | 3335.30 | 29859.24 | 2415.10 | 5069.76 | 6195.14 | 227.81 | 444.76 | 510.18 | 8.64 | 13.54 | 20.29 | 44.49 | - -### summarization (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 32 | 91.68 | 512 | 5.58 | 22445.97 | 2821.71 | 25267.67 | 5587.21 | 11177.97 | 12257.18 | 430.38 | 967.32 | 2238.28 | 10.56 | 16.13 | 23.53 | 54.40 | - -## p3_extension - -### decode_heavy (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 56.16 | 32 | 0.57 | 145.47 | 546.26 | 691.72 | 1754.17 | 2994.25 | 3175.70 | 52.30 | 107.66 | 108.83 | 1.88 | 2.68 | 3.14 | 8.53 | -| 8 | 54.33 | 128 | 2.36 | 648.72 | 2526.47 | 3175.19 | 3302.64 | 6408.14 | 7166.98 | 74.21 | 143.24 | 149.17 | 3.05 | 4.37 | 4.79 | 14.12 | -| 16 | 65.74 | 256 | 3.89 | 1026.90 | 4012.07 | 5038.96 | 3934.72 | 7216.03 | 9638.96 | 81.02 | 165.95 | 197.11 | 3.98 | 5.82 | 7.21 | 17.79 | -| 32 | 92.36 | 512 | 5.54 | 1471.70 | 5761.81 | 7233.51 | 5562.57 | 10498.47 | 12822.35 | 107.17 | 234.99 | 426.35 | 5.40 | 8.14 | 9.03 | 24.38 | - -### long_rag (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 1 | 30.84 | 32 | 1.04 | 15823.63 | 281.34 | 16104.98 | 962.70 | 1688.61 | 1788.29 | 549.72 | 1124.54 | 1163.96 | 1.49 | 1.72 | 1.82 | 8.52 | -| 2 | 11.49 | 32 | 2.78 | 42460.67 | 754.95 | 43215.62 | 708.16 | 1202.62 | 1253.12 | 192.57 | 352.96 | 409.32 | 2.05 | 2.37 | 6.62 | 10.76 | -| 4 | 7.09 | 32 | 4.51 | 68816.98 | 1223.56 | 70040.54 | 854.49 | 1516.80 | 1562.42 | 191.57 | 426.45 | 427.69 | 2.62 | 3.20 | 7.33 | 13.69 | -| 8 | 60.48 | 128 | 2.12 | 33604.85 | 536.85 | 34141.70 | 3732.80 | 8420.06 | 9705.36 | 930.98 | 2125.14 | 2742.48 | 11.12 | 22.70 | 39.17 | 61.41 | - -### stress_generation (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 96 | 81.20 | 768 | 9.46 | 4842.45 | 4635.63 | 9478.08 | 9772.15 | 20945.93 | 25314.28 | 326.63 | 782.50 | 795.30 | 20.35 | 33.33 | 42.50 | 88.91 | -| 128 | 92.26 | 1024 | 11.10 | 5587.83 | 5514.57 | 11102.40 | 11012.89 | 22012.44 | 28276.20 | 360.94 | 812.12 | 1024.93 | 22.53 | 35.59 | 42.09 | 99.75 | - -### stress_standard (20260707-132641) - -| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) | -|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------| -| 96 | 30.50 | 768 | 25.18 | 12891.05 | 3175.02 | 16066.07 | 3689.97 | 9383.09 | 11958.09 | 435.22 | 1225.11 | 2581.90 | 29.46 | 67.30 | 123.42 | 113.64 | -| 128 | 36.86 | 1024 | 27.78 | 13985.42 | 3479.38 | 17464.80 | 4417.43 | 11825.67 | 14850.43 | 456.55 | 1032.19 | 1922.87 | 35.20 | 76.85 | 141.17 | 136.57 | - diff --git a/experiments/legacy_bench_results/dspark_st_comparison_20260707-150649/comparison_report.md b/experiments/legacy_bench_results/dspark_st_comparison_20260707-150649/comparison_report.md deleted file mode 100644 index 0240608..0000000 --- a/experiments/legacy_bench_results/dspark_st_comparison_20260707-150649/comparison_report.md +++ /dev/null @@ -1,129 +0,0 @@ -# vllm-dspark `--spec-tokens` 对比报告 - -- 结果目录：`/data/user1/yy/bench_results/dspark_st_comparison_20260707-150649` -- 模型：`/data/models/DeepSeek-V4-Flash-DSpark` -- 后端：vllm-dspark (TP=8, FP8 KV cache) -- 对比参数：`--spec-tokens 3` vs `--spec-tokens 5` -- Warmup：100 条 -- 压测客户端：`sglang.bench_serving --backend vllm` - ---- - -## 核心指标对比 - -| Scenario | Concurrency | Spec | Duration(s) | Req/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| chat_short | 1 | 3 | 12.80 | 2.50 | 363.90 | 1002.30 | 398.97 | 757.11 | 948.62 | 43.97 | 68.89 | 2.42 | 3.76 | -| chat_short | 1 | 5 | 11.72 | 2.73 | 397.37 | 1094.51 | 365.29 | 743.89 | 782.98 | 64.42 | 174.36 | 2.06 | 3.40 | -| chat_short | 8 | 3 | 11.08 | 11.55 | 1525.06 | 4705.85 | 672.66 | 1269.82 | 1552.65 | 76.20 | 221.18 | 4.67 | 8.43 | -| chat_short | 8 | 5 | 11.78 | 10.87 | 1434.62 | 4426.78 | 718.40 | 1511.14 | 2008.00 | 94.97 | 220.85 | 4.95 | 9.93 | -| chat_short | 16 | 3 | 21.82 | 11.73 | 1634.03 | 4727.68 | 1333.99 | 2611.76 | 2921.26 | 151.65 | 337.53 | 8.85 | 17.66 | -| chat_short | 16 | 5 | 16.42 | 15.59 | 2171.58 | 6282.94 | 997.37 | 2147.86 | 2496.56 | 107.56 | 240.46 | 7.09 | 21.93 | -| chat_short | 32 | 3 | 24.62 | 20.79 | 2754.96 | 8275.55 | 1497.40 | 2944.48 | 3658.90 | 144.47 | 453.29 | 10.72 | 23.97 | -| chat_short | 32 | 5 | 26.32 | 19.45 | 2577.42 | 7742.25 | 1600.40 | 3403.00 | 4521.14 | 169.13 | 402.36 | 11.39 | 27.82 | -| chat_short | 64 | 3 | 21.66 | 23.63 | 3131.20 | 9405.73 | 2595.90 | 5230.98 | 6258.25 | 251.84 | 509.86 | 18.60 | 40.84 | -| chat_short | 64 | 5 | 21.26 | 24.09 | 3191.43 | 9586.66 | 2555.12 | 5387.59 | 6559.69 | 274.71 | 610.40 | 18.47 | 42.08 | -| chat_standard | 64 | 3 | 20.73 | 24.70 | 3288.58 | 15680.46 | 2483.46 | 4798.63 | 5693.88 | 271.41 | 947.15 | 17.93 | 46.05 | -| chat_standard | 64 | 5 | 20.91 | 24.49 | 3259.71 | 15542.80 | 2512.52 | 5394.43 | 6023.55 | 300.81 | 932.60 | 17.83 | 44.47 | -| decode_heavy | 1 | 3 | 68.78 | 0.47 | 446.03 | 564.81 | 2148.54 | 3712.03 | 3835.51 | 38.60 | 41.18 | 2.28 | 3.03 | -| decode_heavy | 1 | 5 | 57.70 | 0.55 | 531.76 | 673.37 | 1802.00 | 2957.04 | 3676.66 | 59.06 | 114.78 | 1.92 | 3.03 | -| decode_heavy | 32 | 3 | 96.14 | 5.33 | 5535.64 | 6949.57 | 5809.29 | 10830.76 | 12546.55 | 86.83 | 382.71 | 5.62 | 8.11 | -| decode_heavy | 32 | 5 | 92.75 | 5.52 | 5737.73 | 7203.28 | 5595.02 | 10412.11 | 12522.00 | 107.12 | 331.08 | 5.43 | 9.15 | -| generation_standard | 64 | 3 | 38.58 | 13.27 | 6705.88 | 13363.77 | 4542.22 | 8273.38 | 9355.10 | 190.34 | 813.58 | 9.00 | 15.36 | -| generation_standard | 64 | 5 | 43.39 | 11.80 | 5961.40 | 11880.13 | 5075.77 | 9863.55 | 11533.47 | 216.87 | 626.93 | 10.09 | 20.90 | -| long_context_probe | 1 | 3 | 26.10 | 1.23 | 333.92 | 10074.03 | 815.18 | 1382.34 | 1556.00 | 265.28 | 558.73 | 2.02 | 2.19 | -| long_context_probe | 1 | 5 | 22.22 | 1.44 | 392.31 | 11835.70 | 693.30 | 1172.84 | 1404.17 | 267.61 | 561.65 | 1.53 | 1.88 | -| long_context_probe | 4 | 3 | 7.34 | 4.36 | 1187.03 | 35811.63 | 888.88 | 1550.70 | 1611.29 | 125.51 | 282.82 | 2.77 | 3.23 | -| long_context_probe | 4 | 5 | 6.59 | 4.85 | 1322.32 | 39893.35 | 806.55 | 1390.87 | 1500.22 | 143.42 | 236.41 | 2.41 | 3.54 | -| long_context_probe | 8 | 3 | 38.61 | 3.31 | 838.49 | 27327.58 | 2387.05 | 5028.60 | 5667.97 | 406.10 | 1093.54 | 7.74 | 17.09 | -| long_context_probe | 8 | 5 | 35.94 | 3.56 | 900.89 | 29361.32 | 2223.29 | 4693.10 | 5345.72 | 425.03 | 1241.57 | 7.08 | 19.83 | -| rag_medium | 1 | 3 | 22.31 | 1.43 | 403.86 | 3317.84 | 696.09 | 1165.77 | 1315.00 | 105.51 | 157.60 | 2.10 | 2.81 | -| rag_medium | 1 | 5 | 18.16 | 1.76 | 496.09 | 4075.47 | 566.58 | 968.90 | 1134.15 | 110.34 | 156.35 | 1.64 | 2.34 | -| rag_medium | 8 | 3 | 20.57 | 6.22 | 1573.71 | 14730.01 | 1251.63 | 2472.54 | 2796.12 | 127.54 | 309.65 | 4.46 | 7.41 | -| rag_medium | 8 | 5 | 19.13 | 6.69 | 1692.13 | 15838.39 | 1175.26 | 2263.59 | 3106.42 | 154.24 | 352.19 | 4.14 | 8.36 | -| rag_medium | 32 | 3 | 45.02 | 11.37 | 2923.96 | 26176.72 | 2756.39 | 5579.00 | 6675.25 | 215.15 | 477.20 | 10.00 | 19.84 | -| rag_medium | 32 | 5 | 42.98 | 11.91 | 3062.92 | 27420.79 | 2626.21 | 5182.78 | 6873.72 | 253.09 | 522.80 | 9.17 | 18.55 | -| stress_standard | 64 | 3 | 21.35 | 23.99 | 3193.19 | 15225.66 | 2569.66 | 5006.16 | 5831.81 | 261.98 | 710.57 | 18.02 | 36.01 | -| stress_standard | 64 | 5 | 21.75 | 23.54 | 3133.74 | 14942.17 | 2621.52 | 5431.08 | 7161.16 | 305.11 | 734.92 | 18.42 | 41.01 | -| stress_standard | 96 | 3 | 20.36 | 37.72 | 4756.72 | 24069.73 | 2433.73 | 4827.07 | 6001.29 | 272.61 | 749.99 | 18.04 | 38.52 | -| stress_standard | 96 | 5 | 25.33 | 30.32 | 3823.91 | 19349.55 | 3032.19 | 6272.09 | 8124.32 | 339.22 | 793.18 | 23.06 | 51.46 | -| stress_standard | 128 | 3 | 21.58 | 47.45 | 5942.48 | 29828.36 | 2581.42 | 5162.16 | 6222.01 | 298.69 | 970.61 | 19.09 | 40.93 | -| stress_standard | 128 | 5 | 28.36 | 36.11 | 4522.53 | 22700.92 | 3405.74 | 7240.13 | 9176.77 | 382.02 | 1087.53 | 25.74 | 57.83 | - ---- - -## 吞吐 winner 统计 - -| Scenario | Concurrency | Winner (Total tok/s) | st=3 Total tok/s | st=5 Total tok/s | 提升 | -|---|---:|---:|---:|---:|---:| -| chat_short | 1 | st=5 | 1002.30 | 1094.51 | 9.20% | -| chat_short | 8 | st=3 | 4705.85 | 4426.78 | 6.30% | -| chat_short | 16 | st=5 | 4727.68 | 6282.94 | 32.90% | -| chat_short | 32 | st=3 | 8275.55 | 7742.25 | 6.89% | -| chat_short | 64 | st=5 | 9405.73 | 9586.66 | 1.92% | -| chat_standard | 64 | st=3 | 15680.46 | 15542.80 | 0.89% | -| decode_heavy | 1 | st=5 | 564.81 | 673.37 | 19.22% | -| decode_heavy | 32 | st=5 | 6949.57 | 7203.28 | 3.65% | -| generation_standard | 64 | st=3 | 13363.77 | 11880.13 | 12.49% | -| long_context_probe | 1 | st=5 | 10074.03 | 11835.70 | 17.49% | -| long_context_probe | 4 | st=5 | 35811.63 | 39893.35 | 11.40% | -| long_context_probe | 8 | st=5 | 27327.58 | 29361.32 | 7.44% | -| rag_medium | 1 | st=5 | 3317.84 | 4075.47 | 22.84% | -| rag_medium | 8 | st=5 | 14730.01 | 15838.39 | 7.52% | -| rag_medium | 32 | st=5 | 26176.72 | 27420.79 | 4.75% | -| stress_standard | 64 | st=3 | 15225.66 | 14942.17 | 1.90% | -| stress_standard | 96 | st=3 | 24069.73 | 19349.55 | 24.39% | -| stress_standard | 128 | st=3 | 29828.36 | 22700.92 | 31.40% | - ---- - -## 关键发现 - -### 1. 并发是决定性因素 - -- **低并发（c=1）和中低并发（c=8~32）**：`st=5` 在多数场景下更优，尤其是长上下文和重 decode 场景。 -- **高并发（c≥96）**：`st=3` 明显更优，`stress_standard` c=128 时 st=3 比 st=5 高 **31.4%**。 -- **中并发（c=64）**：两者基本持平，差异多在 2% 以内。 - -### 2. st=5 更适合长上下文和重 decode - -| 场景 | 最佳 spec | 原因 | -|---|---|---| -| long_context_probe (16K) | st=5 | 长 prefill 下 st=5 的接受长度更高 | -| rag_medium (4K) | st=5 | 中长输入下 st=5 延迟和吞吐都更优 | -| decode_heavy (2K output) | st=5 | 输出越长，st=5 的投机收益越大 | - -### 3. st=3 在极限并发下更稳 - -`stress_standard` 随并发增加，st=3 与 st=5 的差距拉大： - -| 并发 | st=3 Total tok/s | st=5 Total tok/s | 差距 | -|---|---:|---:|---:| -| 64 | 15225.66 | 14942.17 | 基本持平 | -| 96 | 24069.73 | 19349.55 | st=3 高 24.4% | -| 128 | 29828.36 | 22700.92 | st=3 高 31.4% | - -这说明 st=5 在极限并发下验证开销和 KV cache 压力显著增加，而 st=3 的验证 batch 更小、调度更稳定。 - -### 4. chat_short c=16 的异常 - -`chat_short c=16` 时 st=5 比 st=3 高 32.9%，是一个明显的 outlier。可能原因是该并发度下 st=5 的 draft 接受率和 batch 利用率恰好达到甜点。 - ---- - -## 优化建议 - -### 推荐配置 - -| 负载特征 | 推荐 `--spec-tokens` | 说明 | -|---|---|---| -| 低并发 / 在线交互（c ≤ 32） | **5** | 延迟低、单请求吞吐高 | -| 中并发（c ≈ 64） | 3 或 5 均可 | 差异很小 | -| 高并发 / 压测（c ≥ 96） | **3** | 吞吐更高、P99 更稳 | -| 长上下文 / RAG / 重 decode | **5** | 接受长度优势更明显 | - -### 下一步 - -1. 如果业务以高并发为主，将默认服务改为 `--spec-tokens 3`。 -2. 如果业务混合，可考虑按输入长度或并发度路由到不同服务实例。 -3. 本次 warmup 已从 10 提升到 100，P99 TTFT 相比首次 grid 测试有明显改善；建议保持 100 条 warmup。 diff --git a/experiments/legacy_bench_results/dsv4_backend_comparison_20260707/README.md b/experiments/legacy_bench_results/dsv4_backend_comparison_20260707/README.md deleted file mode 100644 index 0845e31..0000000 --- a/experiments/legacy_bench_results/dsv4_backend_comparison_20260707/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# DSV4 Backend Comparison Raw Outputs (2026-07-07) - -## Location - -`/data/user1/yy/bench_results/dsv4_backend_comparison_20260707` - -## Contents - -- `raw_outputs/` — raw per-benchmark JSONL files produced by `sglang.bench_serving --output-file --output-details`. -- `README.md` — this file. - -## File Naming Convention - -Each file follows the pattern: - -``` -{backend}_0707_{concurrency}_{input_len}_{output_len}.jsonl -``` - -Where: - -- `{backend}`: `sglang` or `vllm` -- `{concurrency}`: max concurrent requests (`max-concurrency`) -- `{input_len}`: `--random-input-len` -- `{output_len}`: `--random-output-len` - -## Output File Inventory - -### SGLang backend - -| File | Concurrency | Input len | Output len | -|---|---|---|---| -| sglang_0707_32_512_256.jsonl | 32 | 512 | 256 | -| sglang_0707_128_512_256.jsonl | 128 | 512 | 256 | -| sglang_0707_256_512_256.jsonl | 256 | 512 | 256 | -| sglang_0707_512_512_256.jsonl | 512 | 512 | 256 | -| sglang_0707_512_512_2000.jsonl | 512 | 512 | 2000 | -| sglang_0707_32_512_2000.jsonl | 32 | 512 | 2000 | -| sglang_0707_32_4000_512.jsonl | 32 | 4000 | 512 | -| sglang_0707_128_4000_512.jsonl | 128 | 4000 | 512 | -| sglang_0707_512_4000_512.jsonl | 512 | 4000 | 512 | -| sglang_0707_32_16000_512.jsonl | 32 | 16000 | 512 | -| sglang_0707_128_16000_512.jsonl | 128 | 16000 | 512 | -| sglang_0707_512_1000_256.jsonl | 512 | 1000 | 256 | -| sglang_0707_768_1000_256.jsonl | 768 | 1000 | 256 | -| sglang_0707_1024_1000_256.jsonl | 1024 | 1000 | 256 | -| sglang_0707_512_1000_1000.jsonl | 512 | 1000 | 1000 | - -### vLLM backend - -| File | Concurrency | Input len | Output len | -|---|---|---|---| -| vllm_0707_32_512_256.jsonl | 32 | 512 | 256 | -| vllm_0707_128_512_256.jsonl | 128 | 512 | 256 | -| vllm_0707_256_512_256.jsonl | 256 | 512 | 256 | -| vllm_0707_128_512_2000.jsonl | 128 | 512 | 2000 | -| vllm_0707_256_512_2000.jsonl | 256 | 512 | 2000 | -| vllm_0707_512_512_2000.jsonl | 512 | 512 | 2000 | -| vllm_0707_32_1000_256.jsonl | 32 | 1000 | 256 | -| vllm_0707_128_1000_256.jsonl | 128 | 1000 | 256 | -| vllm_0707_256_1000_256.jsonl | 256 | 1000 | 256 | -| vllm_0707_512_1000_256.jsonl | 512 | 1000 | 256 | -| vllm_0707_768_1000_256.jsonl | 768 | 1000 | 256 | -| vllm_0707_1024_1000_256.jsonl | 1024 | 1000 | 256 | -| vllm_0707_32_1000_1000.jsonl | 32 | 1000 | 1000 | -| vllm_0707_128_1000_1000.jsonl | 128 | 1000 | 1000 | -| vllm_0707_256_1000_1000.jsonl | 256 | 1000 | 1000 | -| vllm_0707_512_1000_1000.jsonl | 512 | 1000 | 1000 | -| vllm_0707_1024_1000_1000.jsonl | 1024 | 1000 | 1000 | -| vllm_0707_32_4000_512.jsonl | 32 | 4000 | 512 | -| vllm_0707_128_4000_512.jsonl | 128 | 4000 | 512 | -| vllm_0707_256_4000_512.jsonl | 256 | 4000 | 512 | -| vllm_0707_512_4000_512.jsonl | 512 | 4000 | 512 | -| vllm_0707_32_8000_1000.jsonl | 32 | 8000 | 1000 | -| vllm_0707_128_8000_1000.jsonl | 128 | 8000 | 1000 | -| vllm_0707_256_8000_1000.jsonl | 256 | 8000 | 1000 | -| vllm_0707_512_8000_1000.jsonl | 512 | 8000 | 1000 | -| vllm_0707_32_16000_512.jsonl | 32 | 16000 | 512 | -| vllm_0707_128_16000_512.jsonl | 128 | 16000 | 512 | -| vllm_0707_256_16000_512.jsonl | 256 | 16000 | 512 | -| vllm_0707_32_32000_512.jsonl | 32 | 32000 | 512 | -| vllm_0707_128_32000_512.jsonl | 128 | 32000 | 512 | - -## Provenance - -These files are **legacy raw outputs** from an ad-hoc SGLang vs vLLM backend comparison sweep run on 2026-07-07. The exact generator command/script that produced them was not preserved in the repository. They were generated using `sglang.bench_serving` with `--output-file` and `--output-details`, against: - -- Model: `/data/models/DeepSeek-V4-Flash` -- SGLang port: `30000` -- vLLM port: `8000` (inferred from file content) -- Dataset: `random` - -For future reproducible comparisons, use the script: - -``` -scripts/benchmark_dsv4_backend_comparison.sh -``` - -## Related Reports - -- `/data/user1/yy/dsv4_inference_comparison_report.md` — earlier DSV4 inference comparison report. -- `/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/` — earlier comparison run outputs (`.json`/`.log`). diff --git a/experiments/legacy_bench_results/dsv4_comparison_20260705_152221/summary.json b/experiments/legacy_bench_results/dsv4_comparison_20260705_152221/summary.json deleted file mode 100644 index 98e8e23..0000000 --- a/experiments/legacy_bench_results/dsv4_comparison_20260705_152221/summary.json +++ /dev/null @@ -1,197 +0,0 @@ -[ - { - "service": "vllm-dspark-dspark-st5", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 5, - "concurrency": 1, - "request_throughput": 1.0665689780673602, - "output_throughput": 257.1817776813825, - "total_input_tokens": 47865, - "total_output_tokens": 48226, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st5_c1.json" - }, - { - "service": "vllm-dspark-dspark-st5", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 5, - "concurrency": 16, - "request_throughput": 7.702621472216438, - "output_throughput": 1862.3398195524906, - "total_input_tokens": 47865, - "total_output_tokens": 48356, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st5_c16.json" - }, - { - "service": "vllm-dspark-dspark-st5", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 5, - "concurrency": 64, - "request_throughput": 9.360404619968996, - "output_throughput": 2210.4595510056783, - "total_input_tokens": 47865, - "total_output_tokens": 47230, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st5_c64.json" - }, - { - "service": "vllm-dspark-nospec", - "engine": "vllm-dspark", - "spec_method": null, - "spec_tokens": null, - "concurrency": 1, - "request_throughput": 0.5918258062472803, - "output_throughput": 144.62151314361665, - "total_input_tokens": 47865, - "total_output_tokens": 48873, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-nospec_c1.json" - }, - { - "service": "vllm-dspark-nospec", - "engine": "vllm-dspark", - "spec_method": null, - "spec_tokens": null, - "concurrency": 16, - "request_throughput": 5.61467992292336, - "output_throughput": 1364.0022670753865, - "total_input_tokens": 47865, - "total_output_tokens": 48587, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-nospec_c16.json" - }, - { - "service": "vllm-dspark-nospec", - "engine": "vllm-dspark", - "spec_method": null, - "spec_tokens": null, - "concurrency": 64, - "request_throughput": 7.339983457334795, - "output_throughput": 1803.9844342264591, - "total_input_tokens": 47865, - "total_output_tokens": 49155, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-nospec_c64.json" - }, - { - "service": "vllm-dspark-dspark-st3", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 3, - "concurrency": 1, - "request_throughput": 1.033805786989743, - "output_throughput": 254.38859000456605, - "total_input_tokens": 47865, - "total_output_tokens": 49214, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st3_c1.json" - }, - { - "service": "vllm-dspark-dspark-st3", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 3, - "concurrency": 16, - "request_throughput": 8.237095278328889, - "output_throughput": 1992.1826785402332, - "total_input_tokens": 47865, - "total_output_tokens": 48371, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st3_c16.json" - }, - { - "service": "vllm-dspark-dspark-st3", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 3, - "concurrency": 64, - "request_throughput": 14.530514004704537, - "output_throughput": 3507.8113858757224, - "total_input_tokens": 47865, - "total_output_tokens": 48282, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st3_c64.json" - }, - { - "service": "vllm-dspark-dspark-st7", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 7, - "concurrency": 1, - "request_throughput": 1.0285012524448527, - "output_throughput": 251.58169136053544, - "total_input_tokens": 47865, - "total_output_tokens": 48922, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st7_c1.json" - }, - { - "service": "vllm-dspark-dspark-st7", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 7, - "concurrency": 16, - "request_throughput": 7.463416900080192, - "output_throughput": 1772.3002941775428, - "total_input_tokens": 47865, - "total_output_tokens": 47493, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st7_c16.json" - }, - { - "service": "vllm-dspark-dspark-st7", - "engine": "vllm-dspark", - "spec_method": "dspark", - "spec_tokens": 7, - "concurrency": 64, - "request_throughput": 9.128647309323574, - "output_throughput": 2218.5807988214547, - "total_input_tokens": 47865, - "total_output_tokens": 48607, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-dspark-dspark-st7_c64.json" - }, - { - "service": "vllm-main-nospec", - "engine": "vllm", - "spec_method": null, - "spec_tokens": null, - "concurrency": 1, - "request_throughput": 0.5806562443589017, - "output_throughput": 138.2571550630763, - "total_input_tokens": 47865, - "total_output_tokens": 47621, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-main-nospec_c1.json" - }, - { - "service": "vllm-main-nospec", - "engine": "vllm", - "spec_method": null, - "spec_tokens": null, - "concurrency": 16, - "request_throughput": 5.052303923305438, - "output_throughput": 1180.976042072646, - "total_input_tokens": 47865, - "total_output_tokens": 46750, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-main-nospec_c16.json" - }, - { - "service": "vllm-main-nospec", - "engine": "vllm", - "spec_method": null, - "spec_tokens": null, - "concurrency": 64, - "request_throughput": 6.851644652089382, - "output_throughput": 1641.8938662034388, - "total_input_tokens": 47865, - "total_output_tokens": 47927, - "duration_s": null, - "result_file": "/data/user1/yy/bench_results/dsv4_comparison_20260705_152221/vllm-main-nospec_c64.json" - } -] \ No newline at end of file diff --git a/experiments/legacy_bench_results/eagle_grid/dspark_vs_eagle_report.md b/experiments/legacy_bench_results/eagle_grid/dspark_vs_eagle_report.md deleted file mode 100644 index c4097e2..0000000 --- a/experiments/legacy_bench_results/eagle_grid/dspark_vs_eagle_report.md +++ /dev/null @@ -1,96 +0,0 @@ -# DSpark vs SGLang EAGLE 投机解码对比报告 - -- DSpark 结果：`/data/user1/yy/bench_results/dspark_st_comparison_20260707-150649` -- EAGLE 结果：`/data/user1/yy/bench_results/eagle_grid/focused` -- DSpark 模型：`/data/models/DeepSeek-V4-Flash-DSpark` -- EAGLE 模型：`/data/models/DeepSeek-V4-Flash` -- DSpark 后端：vllm-dspark (TP=8, FP8 KV cache, spec-method=dspark) -- EAGLE 后端：SGLang (TP=8, FP8 KV cache, speculative-algorithm=EAGLE, num_steps=3, topk=1, draft_tokens=4) -- 压测客户端：`sglang.bench_serving` -- Warmup：100 条 - -## 核心指标对比 - -| Scenario | Concurrency | Method | Total tok/s | Out tok/s | Req/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| chat_short | 1 | DSpark(st=5) | 1094.51 | 397.37 | 2.73 | 365.29 | 743.89 | 782.98 | 64.42 | 174.36 | 2.06 | 3.40 | -| chat_short | 1 | EAGLE | 272.82 | 99.05 | 0.68 | 1467.67 | 3775.96 | 5175.09 | 223.86 | 405.20 | 7.47 | 19.74 | -| chat_short | 8 | DSpark(st=3) | 4705.85 | 1525.06 | 11.55 | 672.66 | 1269.82 | 1552.65 | 76.20 | 221.18 | 4.67 | 8.43 | -| chat_short | 8 | EAGLE | 1059.99 | 343.52 | 2.60 | 3043.16 | 7143.57 | 8345.08 | 245.91 | 528.86 | 21.41 | 36.02 | -| chat_short | 16 | DSpark(st=5) | 6282.94 | 2171.58 | 15.59 | 997.37 | 2147.86 | 2496.56 | 107.56 | 240.46 | 7.09 | 21.93 | -| chat_short | 16 | EAGLE | 1506.42 | 520.67 | 3.74 | 4240.11 | 8031.57 | 10192.48 | 236.19 | 438.56 | 29.12 | 54.02 | -| chat_short | 32 | DSpark(st=3) | 8275.55 | 2754.96 | 20.79 | 1497.40 | 2944.48 | 3658.90 | 144.47 | 453.29 | 10.72 | 23.97 | -| chat_short | 32 | EAGLE | 2461.23 | 819.35 | 6.18 | 4731.11 | 9926.81 | 11388.85 | 243.84 | 571.89 | 34.22 | 54.92 | -| chat_short | 64 | DSpark(st=5) | 9586.66 | 3191.43 | 24.09 | 2555.12 | 5387.59 | 6559.69 | 274.71 | 610.40 | 18.47 | 42.08 | -| chat_short | 64 | EAGLE | 4474.88 | 1489.70 | 11.24 | 5239.80 | 10296.02 | 11922.06 | 268.27 | 452.80 | 38.17 | 61.24 | -| chat_standard | 64 | DSpark(st=3) | 15680.46 | 3288.58 | 24.70 | 2483.46 | 4798.63 | 5693.88 | 271.41 | 947.15 | 17.93 | 46.05 | -| chat_standard | 64 | EAGLE | 6746.24 | 1414.85 | 10.63 | 5637.63 | 10693.92 | 12907.15 | 296.86 | 597.18 | 41.12 | 69.08 | -| decode_heavy | 1 | DSpark(st=5) | 673.37 | 531.76 | 0.55 | 1802.00 | 2957.04 | 3676.66 | 59.06 | 114.78 | 1.92 | 3.03 | -| decode_heavy | 1 | EAGLE | 261.35 | 206.39 | 0.22 | 4643.62 | 10452.02 | 18810.32 | 162.91 | 314.71 | 4.84 | 11.69 | -| decode_heavy | 32 | DSpark(st=5) | 7203.28 | 5737.73 | 5.52 | 5595.02 | 10412.11 | 12522.00 | 107.12 | 331.08 | 5.43 | 9.15 | -| decode_heavy | 32 | EAGLE | 2981.50 | 2374.90 | 2.28 | 13707.03 | 27159.67 | 30638.81 | 192.78 | 442.78 | 13.23 | 19.26 | -| generation_standard | 64 | DSpark(st=3) | 13363.77 | 6705.88 | 13.27 | 4542.22 | 8273.38 | 9355.10 | 190.34 | 813.58 | 9.00 | 15.36 | -| generation_standard | 64 | EAGLE | 5609.33 | 2814.73 | 5.57 | 11057.53 | 22577.31 | 26713.41 | 275.48 | 944.52 | 21.88 | 34.80 | -| long_context_probe | 1 | DSpark(st=5) | 11835.70 | 392.31 | 1.44 | 693.30 | 1172.84 | 1404.17 | 267.61 | 561.65 | 1.53 | 1.88 | -| long_context_probe | 1 | EAGLE | 5556.95 | 184.19 | 0.68 | 1477.16 | 2792.68 | 3836.17 | 238.26 | 618.74 | 4.43 | 8.62 | -| long_context_probe | 4 | DSpark(st=5) | 39893.35 | 1322.32 | 4.85 | 806.55 | 1390.87 | 1500.22 | 143.42 | 236.41 | 2.41 | 3.54 | -| long_context_probe | 4 | EAGLE | 6660.56 | 220.77 | 0.81 | 4777.51 | 9442.45 | 10071.75 | 433.60 | 1230.20 | 14.85 | 21.65 | -| long_context_probe | 8 | DSpark(st=5) | 29361.32 | 900.89 | 3.56 | 2223.29 | 4693.10 | 5345.72 | 425.03 | 1241.57 | 7.08 | 19.83 | -| long_context_probe | 8 | EAGLE | 9890.47 | 303.47 | 1.20 | 6232.13 | 16324.27 | 20329.21 | 606.76 | 2671.06 | 22.79 | 54.38 | -| rag_medium | 1 | DSpark(st=5) | 4075.47 | 496.09 | 1.76 | 566.58 | 968.90 | 1134.15 | 110.34 | 156.35 | 1.64 | 2.34 | -| rag_medium | 1 | EAGLE | 1326.02 | 161.41 | 0.57 | 1742.43 | 3804.17 | 3936.47 | 195.07 | 337.55 | 5.83 | 12.84 | -| rag_medium | 8 | DSpark(st=5) | 15838.39 | 1692.13 | 6.69 | 1175.26 | 2263.59 | 3106.42 | 154.24 | 352.19 | 4.14 | 8.36 | -| rag_medium | 8 | EAGLE | 3676.32 | 392.77 | 1.55 | 4908.89 | 11659.24 | 12566.81 | 254.65 | 678.44 | 18.53 | 35.36 | -| rag_medium | 32 | DSpark(st=5) | 27420.79 | 3062.92 | 11.91 | 2626.21 | 5182.78 | 6873.72 | 253.09 | 522.80 | 9.17 | 18.55 | -| rag_medium | 32 | EAGLE | 13268.07 | 1482.05 | 5.76 | 5444.98 | 10471.13 | 11990.37 | 337.62 | 2204.45 | 20.09 | 39.98 | -| stress_standard | 64 | DSpark(st=3) | 15225.66 | 3193.19 | 23.99 | 2569.66 | 5006.16 | 5831.81 | 261.98 | 710.57 | 18.02 | 36.01 | -| stress_standard | 64 | EAGLE | 5399.35 | 1132.38 | 8.51 | 6678.22 | 13086.21 | 14811.42 | 314.38 | 584.25 | 48.19 | 81.06 | -| stress_standard | 96 | DSpark(st=3) | 24069.73 | 4756.72 | 37.72 | 2433.73 | 4827.07 | 6001.29 | 272.61 | 749.99 | 18.04 | 38.52 | -| stress_standard | 96 | EAGLE | 8833.26 | 1745.65 | 13.84 | 6792.34 | 14000.22 | 16722.55 | 360.14 | 959.71 | 52.90 | 96.70 | -| stress_standard | 128 | DSpark(st=3) | 29828.36 | 5942.48 | 47.45 | 2581.42 | 5162.16 | 6222.01 | 298.69 | 970.61 | 19.09 | 40.93 | -| stress_standard | 128 | EAGLE | 13189.80 | 2627.70 | 20.98 | 5960.86 | 12256.06 | 14219.75 | 416.12 | 2153.71 | 47.20 | 81.35 | - -## 吞吐 Winner 统计（按 Total tok/s） - -| Scenario | Concurrency | Winner | DSpark Total tok/s | EAGLE Total tok/s | 提升 | -|---|---:|---:|---:|---:|---:| -| chat_short | 1 | DSpark(st=5) | 1094.51 | 272.82 | 301.18% | -| chat_short | 8 | DSpark(st=3) | 4705.85 | 1059.99 | 343.95% | -| chat_short | 16 | DSpark(st=5) | 6282.94 | 1506.42 | 317.08% | -| chat_short | 32 | DSpark(st=3) | 8275.55 | 2461.23 | 236.24% | -| chat_short | 64 | DSpark(st=5) | 9586.66 | 4474.88 | 114.23% | -| chat_standard | 64 | DSpark(st=3) | 15680.46 | 6746.24 | 132.43% | -| decode_heavy | 1 | DSpark(st=5) | 673.37 | 261.35 | 157.65% | -| decode_heavy | 32 | DSpark(st=5) | 7203.28 | 2981.50 | 141.60% | -| generation_standard | 64 | DSpark(st=3) | 13363.77 | 5609.33 | 138.24% | -| long_context_probe | 1 | DSpark(st=5) | 11835.70 | 5556.95 | 112.99% | -| long_context_probe | 4 | DSpark(st=5) | 39893.35 | 6660.56 | 498.95% | -| long_context_probe | 8 | DSpark(st=5) | 29361.32 | 9890.47 | 196.86% | -| rag_medium | 1 | DSpark(st=5) | 4075.47 | 1326.02 | 207.35% | -| rag_medium | 8 | DSpark(st=5) | 15838.39 | 3676.32 | 330.82% | -| rag_medium | 32 | DSpark(st=5) | 27420.79 | 13268.07 | 106.67% | -| stress_standard | 64 | DSpark(st=3) | 15225.66 | 5399.35 | 181.99% | -| stress_standard | 96 | DSpark(st=3) | 24069.73 | 8833.26 | 172.49% | -| stress_standard | 128 | DSpark(st=3) | 29828.36 | 13189.80 | 126.15% | - -## 关键发现 - -- 对比点数：18 -- EAGLE 胜出：0 个 -- DSpark 胜出：18 个 - -**结论：在相同负载下，vllm-dspark 的综合吞吐优于 SGLang EAGLE。** - -### 延迟与稳定性观察 - -- 平均 Mean E2E 比值（EAGLE / DSpark）：3.09 -- 平均 Mean TPOT 比值（EAGLE / DSpark）：3.21 -- 平均 Mean TTFT 比值（EAGLE / DSpark）：1.81 - -> 比值 < 1 表示 EAGLE 更快；> 1 表示 DSpark 更快。 - -## 优化建议 - -1. 如果以吞吐为首要目标，当前配置下 **vllm-dspark 更优**，建议继续使用 `--spec-tokens` 并根据并发选择 3（高并发）或 5（低并发/长上下文）。 -2. SGLang EAGLE 本次表现不佳，平均延迟和 TPOT 均显著高于 DSpark；如需进一步评估 EAGLE，可尝试调整 `--speculative-num-steps`、`--speculative-eagle-topk`、`--speculative-num-draft-tokens` 或更换 MOE runner backend，并确认是否已完成 `sglang.compile_deep_gemm` 预编译。 -3. 注意两套后端的实现差异（CUDA graph、KV cache 管理、调度器、draft 模型架构）会显著影响不同并发和输入长度下的表现，建议按实际业务负载做最终选型。 diff --git a/experiments/legacy_bench_results/eagle_grid/report.md b/experiments/legacy_bench_results/eagle_grid/report.md deleted file mode 100644 index c5ab5d7..0000000 --- a/experiments/legacy_bench_results/eagle_grid/report.md +++ /dev/null @@ -1,6 +0,0 @@ -# vllm-dspark Benchmark Grid Report - -- Result root: `/data/user1/yy/bench_results/eagle_grid` -- Model: `/data/models/DeepSeek-V4-Flash-DSpark` -- Backend: vllm-dspark (TP=8, FP8 KV cache, spec-method=dspark, spec-tokens=5) -- Benchmark client: `sglang.bench_serving --backend vllm` diff --git a/experiments/legacy_bench_results/sglang_8card_max_throughput_20260705_030839/summary.json b/experiments/legacy_bench_results/sglang_8card_max_throughput_20260705_030839/summary.json deleted file mode 100644 index 001f511..0000000 --- a/experiments/legacy_bench_results/sglang_8card_max_throughput_20260705_030839/summary.json +++ /dev/null @@ -1,58 +0,0 @@ -[ - { - "name": "sharegpt_concurrency_128", - "args": "--max-concurrency 128", - "duration_s": 155.26, - "request_throughput": 12.88, - "input_token_throughput": 3710.5, - "output_token_throughput": 2690.48, - "total_token_throughput": 6400.98, - "mean_ttft_ms": 410.55, - "mean_tpot_ms": 48.45, - "mean_e2e_latency_ms": 9696.87, - "accept_length": 2.6, - "output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_128.json" - }, - { - "name": "sharegpt_concurrency_256", - "args": "--max-concurrency 256", - "duration_s": 90.65, - "request_throughput": 22.06, - "input_token_throughput": 6355.48, - "output_token_throughput": 4608.35, - "total_token_throughput": 10963.84, - "mean_ttft_ms": 509.73, - "mean_tpot_ms": 62.71, - "mean_e2e_latency_ms": 10869.76, - "accept_length": 2.6, - "output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_256.json" - }, - { - "name": "sharegpt_concurrency_512", - "args": "--max-concurrency 512", - "duration_s": 71.39, - "request_throughput": 28.02, - "input_token_throughput": 8070.04, - "output_token_throughput": 5851.58, - "total_token_throughput": 13921.61, - "mean_ttft_ms": 1343.73, - "mean_tpot_ms": 121.2, - "mean_e2e_latency_ms": 16176.14, - "accept_length": 2.6, - "output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_512.json" - }, - { - "name": "sharegpt_concurrency_1024", - "args": "--max-concurrency 1024", - "duration_s": 68.21, - "request_throughput": 29.32, - "input_token_throughput": 8445.53, - "output_token_throughput": 6123.84, - "total_token_throughput": 14569.37, - "mean_ttft_ms": 13265.46, - "mean_tpot_ms": 105.36, - "mean_e2e_latency_ms": 27566.99, - "accept_length": 2.6, - "output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_1024.json" - } -] \ No newline at end of file diff --git a/experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/length_distribution.png b/experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/length_distribution.png deleted file mode 100644 index a511adb8662d66ba2acbb8a6c81abc4970389b23..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 150830 zcmeFZc{rA9`!;-u43$z;hzymG$j~I2%a8^VDnnAHkg3S56vS9(4n}(XO>3yeH~d4|@qnJ=@r$O8PR91J5+PL*yCI+fd^ncdQ|-!j{#JtH-! zKsP3PBrPwEH8;LTUYq6o=p_%&pYOKwG0`#7kI%b2klFeDfx~G2W3@N)Uma@iF!9m( z{g2nL)%nHyU#z~hUTnNGM&p2r$|3h6kNJY)!UX;o3u{B| zYDP^ol$FQxx9L1(KPSPFWA!P1G@Zr5`b+8c+i%Y$>!d{$7sl`AqMhm3vv;q8K!|bK z4d$5<^?R1hPnJe%+_=7c{XpS1%kwQIS0!rxxCh)^CXI)x`C!?K73EDS$2Bty_kaBK z>Ghj8j0bM5yYB1D@Up0A?0c+dPv5)nnUP1Ix%b-BDcxA%U0AqpXn5H8+&Q1p>&p{d zCI;BW&(hVz?0cL!94d4?k;k+_PUG-l&FAOT9EU!xcAXr$Gw^tkZ|l}o)+I8Q1K}D1 z*YflCR)>oPe=c%0bFoqG7cxA4@7}$c5&xQrAN8~nkt?pzGgmiz$o7eAZ50$$ozc&C z8hv0md)C4t<=06wE2|Z*uCCh2$7o`tFO@z$`Qqo-w;{o88FNwxuM9g#>+~;Qm8_kz=2VW=%`3mZy>4w?ugK``bRuJ&xo{s_ zU0q#7jmYU7jg0wY9Xmo7jLpp%PoF*=Dr&&_;r{NbJiA`og}Fb$pL6FV4)Tw)#pEs|2=&dt*L{(M(%o*M#w6vKQyJQ`I)X@}P z9&VTr`l_n0&-Fmo-T(2?*t75MZD*JBkoi^B^NWBlbCo&P)z3 zkJPx=R~!-&GPW@98ok$n-mE72^{>8ncE7%=VR!UZq?|j1-~IgZ%C+4WKX?E9Ql^({ z!@Y8&45NdCgS%7UGOyf=T|I*tC6$qqX4*MR>F9WQd8ygS;W9+rvn#(%(|XO6m#V3& zFDor8TfK2(P`H>;4+2r|*%=yr1B1-770aAmTm}knDJx&ODoM6kp2PQ**eus0BG~ur z+h_9j?iS}?UzyIHJaHtx zpWWd;fU(dk(8LoD=5gwu#!u@`N;_y>i4$~WhuuW z+wZ!qto-KdO zR)dluSmGZT7~JzPSy3*U-)xfpbKk% zD%(=+?F#F*tk*~&V?7ni%PTASWn{RDi;E+6S#zp~3VL^UpBW$Rl5705J;R_-z`f*V zVBqzP46)*cnG2LxY%Gt<_zxN?4|%d}aBh6JoQ&J_G3%IUjft3IBmX@W$X)d*$G1{m z!NIG(y^jnTo^Sa69eN5+wf`oSpdW?p%F^`@6=;)c8bs;+)DENrc)QE|YXO)Bx&?>lzy z_6-bV?&#>K%QR7Ws+YM0si)b=Nv^=li*g+O%$=E;DQHk2Wz&)8y>^?bTAU+~PQk@L?O8++qEov&Qs6A{tq>fDS%-w;X5 zuu@f0a;PJp^@UmO4O9p#Yina0n;`qX>NtzWM-e+Oth4$2@+Rj_%OAOAJSVX^q%RHl ze6NiyYieREbQ-OUQ}XQ|9%e@7i-?ST-PXp5s4EjInIkKxwtJ)byZ7%`kiTk4J6)f2 zXg&3+;?{b*-pat2dy=d&ace~`6VEQRisls-(w-6VdVViDI>4s0fN#eR76Ac)kGU6l zTArRn@!)(_SxIkdYHI7?P?lxUX#4xyVeEObZ}0(%^73BAg+~(|@Px^#l1oHF@A>xO zzMxUDj1LRSnUCSUhrImN7l4=IWyXGfQKICB+p;xZI*;j(j#dV4K+$GEY|@yRn6Pcr zS+U<`{B}@64Ho6+>}W{>>Wi(Nop)p;2g(TDs#U83c;q%>$6B`IjjE@@MR2|A#SE3F zMmoQI{Yr-q$tx%jZuXp?*;yIHz1+#kiA`9W@!Pj=wwEqde9SZ%8|mDaT@@j2La(T# zMCxvl>s0B758E4mi=EC93F47^(7%E?N>5Mk;l-|Es^QVJ8?t83)E!;_wc7*^yGD8# z)CSl~RQg}~b})Ej;Kz@8@1u4u-?(w(v#czi>(`ee$6rTSoy#{JyubV64s6_}WKCMs zOT?uL0=7OuWfj-H%OZ_C!Y5<1E#jliwpY&jzkL<6(0s#%CS0W5%~4?ZI%?9%aww?i|^iri=qB) zZ0M4$C`dgyAy$Owia-ANh#lk6{_^dcfPqp$Ufx513(m!Rv=*(& zfNiI9tQ6$sse=LR-U$3xKo}B{8bU#UNw>N@7rBvDw=7#Px&rGRDsplSf>@{U>1N8R zJ%<^^YbiB1*?v*0)LTGDM@KKmisPQh$-4=kL>jg#2?pKgVP|K5hhop?NvTZtSlTHkrq7IS((mbXOTGB5BRfr-@M5b9TQU@ ze_*-Q$7g(v9^b#__0PscMfuAu&apCcN_nNFZ5I&{IhJjCLgZAIs=vbKty@h{lt`_x zva%vMkrd=t-n5oeVm`0%RpK6V&P}PhtAc`pcI?`9qo`;ES1BJprnOS-`=WQwxS zpEug)>kk&*;o0v(hr|agCuP~h_~px&Q`v_WizD~D?t1z1rBusyx*q#D$GzL2+xk%yX2zA$AzClTq$O*iMnEpGL32jx{#Qc^uJn^CeL+hyzC0fCe}45PC1&8-%{Z>9YiN|u{rRnlC5gCyzYGyHG1yuK9ChdJ z-K!|xq|3m9YTY;ToL`F&@bK`UyinGj1jIZ#AL7t<)F4?u|9O_%QPcu~!%>M^Dt5)o z*NbX@x)&PiL&_8LhFv}}G28)xf!K zJ0}Jz0U8NlzGL0tsB_0MAi+AqaDSh)sm102ws-FCKPU_09b2Rcu+2&7QTpm7^ zauoHTNpG}pw3wChxpu8grC;~hvD-UPD5!FLRddVRx9rRLl-qNxms0gbu1=cy_U8kK zeVI2r%Jgi{y;wW`kt)X*UvkkuFpG-Fw0`~iZ0qw9k$Nvo1M^&`(vdYH+N7^9Tf2s2 z<4=n~#1i{?sIzEGBPTz!x09tjnA8cl7&_p-I6oyki)hLp_jn*_7F3z3yY)qDXnJK4 zT9wjIpH7<0q9&(U8)cq7KqC*hqG+<{Ix(uXSk=_j)Q2C)T!|k)ndU5K<_28qzhSpc^8U&B*{K5vBuQ?EiV}Bs_oU0f znjf-Cc3x;{vXFam*4UVj@&ly3aH8wOhr^VG=hnts{{DAo9PrGOtQQs*;+@BOA|E(~ ziyGMPKWI`FbmC9DF8U0S*>DdY#92aRZ(Do&Eo_72fVYTGI?DL$*?7ASMMcHL`Hmvj z-74RsKJ8`XD@XI~lbV`Z>HfCQeyn71{%BHXm2(Ull0GMZ;+ zG&Iz-u>XWUYf59%p>U0o_3WIS{@5U9E<<0+e3u_ObSNw*!>DAPVxqh638%w}&d6eZ zmj+u>hDr>JT$XqVo_G?fdgMq={j^~89y|WVQ7OFBQ@bg0=Y@Nf?kHH>S{J*XH)e!} zvXLD~BM&^5k`~e^ZusnsH^mFUsElZo%E{17xPe4}+;s>y=+}^R$ZSTs+Ij!|_TERR z5ALpk!?WZ4o6sjEwcah5s(f_#zV@Uiwz)4qEk$1bt!?y#?sY|IZ zJw5zkNhLvb;}Lru9%;+X=Lz*B4s3kz;DKe;yvc;*wzij#caj^d&OSfmd;k7@=ilD~ z)%U5WsL;pCOR*Taja}d4K9l7a z6toV2^DKg=Am%Wn*jH#d3eN@ELl>H?72nw#FNdG9|~vk z__p563r)#!{%SJ3hcPK*M_@MBfVD=KaRqTB+^W~55rzEuN=QyX5%Exmad zZDi-E;F|(%b6$I`+obv<20Xs%OtUjFqdf5M*`voL?Wn4qs&klP0kGEP+;lxUD3fK; zy`O;>En5X}f&k)j3+}ocWWa$`oivFe-hFmGKI3JsN$O$cXdTJ{ z=AWJ?<+)5eITC!oxaj)z>v{!_DU8dOtpT@CT2-YIx=r^8#R4oD@7Ui^T3@e;YJb@0 z#tr|UKW#Hzr)-RY11Z0I+fP3k@KlS7{fE5*NM%K_`6048SOqP6nhXv*K4htxg1Wf%VT+}WP;n?1Nmepy+=j$-%I z{qaZ&+`_{SQFRaNkU#_5pKp51#0)^rLir#&N61`xPVPxnLxb(&=a;Kdq}cFHOS~c^ z&iiW>j>~l~VVM|cX5D)90UPeE2l?vkU`v`-oqFd|XE(Q(QkN$uC(nX#1AC&4Ce|Mv zuR0BV#U^*fYb=T{-Dh>$?k!Xe2iUhCwLZ5y&bZWvNz3!Wepi7ovq_Ue{vD^{m6c4&+|YZLg<%A|oTGyA~I=UcGviJUp_qr+>W@ zOUjv@8m%3l``vTtpt5pk7gF@0nbUg6>zQV?8&`94$E3Ln-d#p9Vs~a<8q^_N%IdXi zx45~vCDNI0+qP{r2S-YS$B(+Wv-s14c<4cn=HInT2Uy}39*SD)QY>Omf8*oGjfJXr z_|MMIO!R<#;s-1CgmujtexQ%U3-xHf{L$v$iHHyc%HHapy9*T5;=*i3lPKa{-I+`L zta47JGPkwIIT`FN?eD>I3w3!J!C`eQ8#juk>*w#p&@J|Q;R418&yzlbhqToA6G0}n4i6lM?>-6Six~O zZTig2)CXL3woUH&ck5D0fc{Uk-D^~)A>EmI$34SJy9-^#MR}J7@A(A>-2#A2!o@ ze!tFk{D)?W90zN$)cnU{w?Fp%znx!SUD|_^Pqy1j*Qp}}^}LVSr;8kXtWR$*$0ks= znKqr$VI`@N6JouxfZ65G&%gVdO{}b*?s1)b+SG!D+_LWWeV-dc4=$K~f9RLL?el0b zkK8&YCZ^Kb+M_Er%ILLC_6EzDx-487linmONW_LKW=lRbb;3~m&v(x>sEE}-evwgZI&%h zQ+6rI$jImyFdp&TsywqiD&W)ey?2^dMPSP}7%;cAv_u;2`F222$#m6)3m3%Cynh#=m#!(ITeo@5jlC-pv9rm#oy<~*RSWqZM)_B z9l8-7p9Vs%<}fkmb5C(dnkHKvRa09Ho;g%@y@a9j+OIx05EGU!yHDt4TZWBCxh^gg zX+KzC&<6{V?eqyGp5c?V`czb)3NKtRca%dv|q!fv+;k0k~CqW@wHjEv? z&iE%my+U|gPMZx25f?uGD4?=-r8O%jc)9@0rh^YpKs^ zI|F%-(|!kk{k%E1sY$#3=}F1J)}{U?8_-Jv@1}NE`K{hWcYD{i&%eLdHn(VY1a`i> z;*c`bmVKci;;BL5uG8~6D))Ifii(O}Jl&-Hd5e^}%bPy?`g{koKsnJq$AIg{g7TAR zJw<$2xTPIgC}pI+6FKMsRVj-XmdH>?2Y=TL+coCK?y-xnp7ir$G&V8W)C?;8mU_%z zvdWZ8#`)M_F_Mjw9Vp|leLjs-+h1JJxx~bLq2=kF&lzYc9dc$i`}~?UMd3J=>-w|X zyGgV+;Ngs_xp~Xe$m$Qgiisc2Jaj&K&Mj>{9vs@few#U2_nG80htbdb`vo2a*zs4a z%3HT?9Z9Rg*0C~I__I56tnWAER0Z=Ky2(FadJWvcTF|F=YSh*PsWY60s`31zq_}VQU(zcmec3! zbHl9mfm-Z`DANioO0v|E48tO=wtIgu5C80~jMb|xNQ;f{u0oI0JutA0*Lwc-Hr@0y zVDf)-7Vai=Ee$PwcbV_1$dL;uaUiZWg+}Y*u8oh5U$|mH`bUrnG+r95wVaZsD@3+$ zKT(~bpMMpwm7;(G=pnF#Nn@KeNM9wwQFUYNMW|cN8+Y&6aqRqI(FzmD3#X!u+X~`j zWo6CVvV4Lb0pdJHaW38i)X z_}KJVuhqoq%o`U>M{gl__1wQ$TyN=VXJ_}LDTS4^`)Jg9$NL+lPae_@=?E$Ws|q?V zRZBGdUD8g0`T2Rf`I$@H6MBmvDQPJrD$$v#(Ttp$hYHsSn*&&9p`QT^G6o*!gb>hP z!uV)_NeTE)itx4rO(_WtSekXDozeLomli!PDF)UrK*KVhsDuP-xiNMsuMTy($R!)0nDP~v=ba%f>e zK|JVf>iOc7He@V^?=_S}$+!NxINB#VX-~8sfT~5AUqXTMA{|Pqb;fkD?LdB)&hsgpcPmX^Yhuo5PB-p13d zg;>eD^@w)6&e(pj-odTn;;Zr8mfl{s%HF}Dam2^|N=Lx(R*i^B^i z@&qIu;Y-3gPXz;85vwnnN6}-v24983t|f9X2HDr5s1~67gi9yT#TuaPTAf&`Zz>S^ zq-EHjO(^~{&y0k_H;p%aPvk1*rh9{*y?B93dou1Z&qkJ4Q&SU!P$j;rxdrJ#3F}Ll z2lXQnEGPFVB&M6r1Nz#*KARz zDD;c>`Y#`@Vq|1|ReqCI^YIaG!Um$MxpWFvN(%_kqkLOV86k=S-4BTYTY_Au09nyJ zG!%f#jt66MD*Ho9N6>7zkw?m;iYk?SB6S@oKdL)K-Tc_gRyHP{0hEvaiAVhm`iD&} znwZey(y1pVo{$!(+iTu8S+r%PR9f%76XNmhdi31kV{RJ*x%RGEzE0#7g6C34s#Mg% zCNy2W)4{rQeO0ZiIz}RqBDEVB*|m#n{~#APKL+*2Z&J4rquK+R@AZ zo>g0I2FMZDc0Ylw>O{GdBVR7mvbTq$jX|KW@3dqCinsv=g?nnKw}RttNQfPFkv`9% zaJJE5p#BSPawLh0e;TUf@Cu%l-qf&O$=%7zPj=P$>QIG9iSuwzfMe130dg zEX*ti+Y}*9vWH$=PTC`$^)#w4m=hm85)RwCb!!4Ta860L##_O`t$EwsO)t2B|DnIV ztBk01UW>(2rgItba>x#X+R3*YV%T|P-Hy4&4SxNa*dz-f77c6dFZ~9|UAqoq=N(Hp z$UHvMxdHq8LraB`p@gKQ67+>kbO+!kyHSsjEm;hU-Gn|=-a$4Ld(j#kXWSP1Iv|6m zqN4D{3x}X8k*!tDIYm8}GE$DroZKU^u@^n$kA|dMI-jb?#>Ono1nb9(->9yZ2ON8a z(veh{{7cKGs`Kqzp~m@r($YkW1(17plHqq81o0)5Y3+l+y$;`~H*-aggx<*&n#4v$ z33m$h{`ircX>HnW&!3yI+CWV%D6%sy9L?76pQ9y<0Pm|H3zfmCK=cxlhRr(*c0#zl zGZ?v+8t&?id^=Q>`0?KD+g>0^18KT~4>R%=2{eBYjU)1E{DE6NVAVL~mj8M;7~top z+6Di~m%hHGVq#*HMJxNEdm^t1=MQYCaE<;8x+=?zLi?F1M?eR75!~*1UT|PtPCXo_ zP1@BJD_OZj>TN{Uz`%g;2i|06T3XtVA3whO@PV0o-{o=`3$|_19=Lz!&T4&q{T<*? zKT=+%?8_YQ#Y3?K$9FV1*E!;U=}kVsth21qL%49%mJOuHFU(A+quw zTdPW}ROont-W9JrGeVx8W%8d668#!JSQ1^%0rBP*WVAC#;b2>YKvpdS z(}?=G9KlhxyUUH%%#l$dj5$$O0=u{uV1@HyUV6S z@NicyicidlCvfe@7iO!z+V4FW$ax*)6yN5}bhyBV7#helHTH4?kFL=^3{8mw>yNF3 zXeS+fo9997YKr$T%~JCk`I?&nlK8dFya4u~6?}7Zor={=->8aLxMTBxhT{$UZW?fD zOG(hlBL@y1B*5PfQa)7V9P4&^PfySC@^aPQCx_R)crGH5_H(i^?93BFPqP9tkqDM^Nd=^PPblM-g9LQ@J>idiV*Y;U(igns`u9|;-Av( zB`_I~zeM{14Vm6|70*GKUwR-fjOKuo`d}~j>E=@N+4+2+o`&EL1p!N`YAnFkt-Yw8 zXlO-1rTV&syuV%VMMFy^x--l|lt}v35s)eBZrSx4Dzy*b_G{=vB!02X@Yi``aV<2g zt*s?HeREowU}H+w&Dcm0O(Z`mY4iN-LA&ooyBbmlfyCvz$+M##JwK zwAQY}VT~Ddz-fGu9a?m^-ye3{+S@BSI!eLna82m=qlc)zQ0aBjXD9P3IT}o&y#@a| z_FPOL|KVn7Ro_MHXbe7H4xxdT3Sbvsg7SC}qz&QeI{R0B6+PYf`ZbMw8=Egj`0={& zV^1f-J*p+GO>a*(CB>%y1Si)};viX>pA^B|i)8l4GODx#m5ZGS*YKwB@bEx!pvCjL zj$VEI{K@c>@56*ol)_`v4T;rraeh5y*w&Miu7+1822MO`U+cudw*8cLioTuky-!(g z1#Xm%pL_=5HL<)-{Y*w^XgDmX}hpBMX4f>T_ zejtK~t!dRpnODdr$Dp=?ge+}u-*?(cv@h)a=R^LBU9b?SOR5WgR*CG?yd6He3E$JX za}5j2!%Ra(#jE2dBR#AQi4DT=*1GM9o@SFyy9;wJcCF;21OB*y4fzlV&DPN|)N~1* zBhpPn4||)N=LkrAuPlr2h!|#G>%h^M5euJ0i^#V*=^J$VoUyRD2@_fGEwN&G__(5^EbjISbvKq=oE+)A z1%@E4P3T_aPJm8=VX}rzj7k{j=}kFLYw7d23)|6mmrU6qlV`iEGF!*`Mzms&X?E>V z@s?+viyr-5p9~biIo$(kA_)j~9g9iaR7UO-Yf&%7&sVd>bjaK}-B!*446HSyyv)5Q zoXzgq)8bLV9LXB_#Eo!xs%EemtpmW6jM*hr$Yx~AwthcHMR|EQw8FigOM1@!z&8H0 zu*B;8`RizLDeCfY+k|FIV2(xE;uM06(OrZdehtxZMLaO=Gu!1g4{!o}N73Uqf3lD<~wy^!oK{%XZVt zy7xqDPKsK>~)J0vA( zDK7Zs+J{SKNA{tby#{;m)=o}#DU_SAk?JFhoNtENzaInxnP@WFSvZxjai3VyqJqKk z=poW;#ysh$u3m=wQUdo(_*vks&5aj&-V<|xeA@+6r1q07i%^5@es(KR7IMjEa{+G$ zX1MD{3Y6L}1i^TLoN!<`r@UM_TF!H${ABNnf|19fLOmVbPQ4`(djn znV-C0Iq6TWdKjDe?k2i)W6+dh}gkisRj$uO@!$pbP3a@amH$ql#0t=(a=A zf$Y?K>6}@~bH^)-=OekRcbeqjkP7HEs9(}@ z=}Adjg1Gh)vj&ncp*T5v9V7qD3}$FzPf(}Ub z$_`|%?VZNX5CK1VzK}(6c#wa;d!@O350t~7zkVsAivfA$_*|Lp_M}oVSR1N2l5Il# zpl-UprfbZ`wQH{vz)PVMX8_TcH0(chw*Y_&77fY9JO`WFY)}rj-}M2^kd`Svrv?oQ zguPWH>|I^I|Jb7(`;XBY+>SW01;e&t46zHkyeQ-w$Si?i6pe8kFq#B8KAwb*a>->5 zlmrb0aOV%-wtD2E(@2K`T*ky;_T6$3tPFTJ2v@5~_Q%(*A$COIEadCeYt~#vgb=Fi z=j5c$pLl(+OT@GXf&fYghh`scB*65B$z4f&%XZ4zoF5J!+wHQ1oykJqEx?1~CZNPfXk#y8ZMzAP;RzlM5Ga!CymB zmBBp0fKZ!&-*N9Kr7Hjw;&+W5P+0%H4V`SiO4!EzE@z1Qcxr^3tQ;CNK%#Y8ESJ%* z#g=!S1I95}wrW)sI>j#_NZ`@?Yma4SKA@<$1bk}{&~+Rr6R3!5s58h^>S{Ohll+## z2g3^JTLq8;CC?9!?)`)P8$hhTn?Nd$@J2~hYyE|=Ek`-IgQx?mF5@!J$gpZ-IbxwM z$NF?q4m~|R=|qrJuOTQM2b!3mL|5VYj}0_$BMXI;PBu0I&}?f#h{?km2hT(%yyJWJ?Mof1efMrT60a&u z^(fzV{S8|n9TDd3W1ih6X#1i4#&1bYnl!-2LA-ZAs%ixCo<7ZjSsf2!W*lgGZEt$ zQUMY92HmqSc5UqPnB9)2;)`YpPK7myK#Xd3gi4~t!g(eYc zG8jxTF;A1s6c7+V-0ESw{3AN?F{Fz3OU|Pj5uRh+_EVw#;tsCqXyxvhM*YD;6Jr?3 zTc}gSOMioT!wrz*wA4?yugXASoSxk}W5viHFGFIBhWDK>h|FpHo_3oYT;flDBcuYW zGM_zz$qH;>XmwYS>sCZw!dM;*F!7*WmU#U@CJhSO?!$tIza}Rq=j4O$z)~4G^9W1y zuD&`ZC%`O=yG~l+UNW+RA=E6VV43G{EVk_ZkDW^?eJbfk_^x0W7;l1=%$|MQ&%Vv@ zT2!SWBcI2@A2S+xA{`G;Vq-B$O1{ttb_FPk*{FMe63Q|Fq3 z`!v%4a}Wo|WWYFh0_1VV(|Z*=~* zH7l|9l}zW?~B$OuKp1Xk92eS2M;os+E>fd@1)cqBZk2D?g1U}Di~ ztAf*5V*W@7H>%>@$q7t}=nrpmFM005CT6IQe0O}<1LCiL>%&Kx7$-u_Nhy*>3$YGn znd$kq6nd+-Ul6R{lS9=Icv=zn8w?FDP7|AG5dmU z&g{JtMUcXYPn9OIB3GhPD{D17yCCS#h`tNNe-{nUXrudf!Ae8&x59mWp)X>t`j9+@}OvK>+2tZrQca4M4 z=%}VDZ{&Vu*!83~W}HaXDedeupja?R<*Po6w9S~WpPpk4%U8`zKIdNOVz~LUhBR;SWGkR#USQpGXYWPjoYa$TS3MVlZzy5oS3)Q1CW3EO!N;Ky3Ezdr+$Kk6ot2}|aFEq8Nw*DP?fh}qbV564D+js2nt?Abr6O!N#e zQXn<7o}}8Wr~;J^t|$TGEKy)}ozgitI9_A@5w}=k?j5M((|`Va0TTXXSlfU{4clrJ zuBtxE;y7{rqHz8@ognV`;~0SG=>pa*``rWT-`v?`hrS+0$vEU6$XZuJLRew1xr$UL zoMsXE0Mm=;yZlLy045918&*zv;vR-548n`Qu@~(Vo~<2}9asseO9M>`af;W^;CTYT z5XcEse52;xGCqF(nI*urqC5AHTEp^G=hJa2?9gtbHwk(Tlpcd7-=E-ah?0sht zgPe~mb6>nDLyHwiAU?VwVt5(vtO7i#si`sVDkk-xV6UgAb1~?(GJhnun1h}D71kD3 z>vCMmSodpM?F>T$>&}iKI~(n^Q)>w;c6I6UpB3wkUO`_*n@(l~2*d=VVuE21X*nB1 z%g-_%;b)oGFM&bneTys?H@DqL z$6hYXj$)fC0e%=;STOxN0sw)g40anh5Nj6xeCHw17y1YbG|cXvo~yv* zUqFp|xPw;HliRgI&nVjhtkLJ|yBf+OpyrcIytwgF;01>LXN4+( zP+WmpMplUsc3{u%Su9?%OUHI5&>0(?Z@8+ijs+tR3fOl)k?)B2QF_r(d0glYQ=F;7 zUZnZ>T0fWy@HBmp=u40LII{RG+1IA9-COF$H%bMjbcR-Wz`y z&9rjmL3FTuKCjSg>=e*JAzmPf8ZSX3|KMP5QRdC(Hjx{8fB%c z6WeF|zjNl}rno$I8Rs4Fm)?Nl$PYpd!q_X|iZKYs7DL0*4403qui z9o|6Hc1#_>?xv)nv4Tw1;TfnN5+tHLB+n*?i#PpLH_T-TkQ*twiE)^yofn>&e&Eb* z@3Y^FA(N}1F+9FL0Ny7rO@&Q9ui$d9{rYBg;3YC-h^_r?)1<$jpZ&FeM&%QC|JzE! zg-VOig~~;*$hb#Zx(Btk6bXo^s_sr9WHs`QIDgQ_z#vh2KfChb2zXz+a*z`Y|b&Wrqpme$?kLIn;cnwOr7T@J3U z@&b)Up0dlA_#NH-?a?H)=iA>Wd1VV8H(KPG5qb-JP6J2fK2;pATumtXnZ zSfXg>RDsTDqJvdy&Bm1<+ZLwg#!OyS9yBWYBeVGiZDC{JmDVb+%)nw7CGwf$Z$Z>F zI;~ed6u^DEX2)lRnO703$iwvx3(usz@?bdfzt_@F(D(o2xdq})P7Hp_q&97OsKn6s zJv8_0L9gTZm%lfntem(n!0EQ^j&FZ2!V=LfxLZWTtH^bt^H%MxxTQav4c(`%{`1v8 z8Y`R``e{vM$*(5Thev}q_;+JgSvSlgu znY^|(8;0B`@b@uqE@sjquKEsYt$_i^v$j2FX+XeuYizB8<(i+;oe}$TmVV`O!0$Jt zK^G&#$o%|@T?Tv&Dd3OYrhn4lK>~7V7c*+GXv&uV=N7eJC1TPT1Jodu5HEL*^yPk} zfoNdANr3)GQ-8^KV`Hztz|RH0PpSsd)I8qsJKY#h^K$`N$k<0{frTOoI>1f9QTEp9 z_{i$>gdilt(_CBu)6>r6t3hHkJLUQb%AI|AZQNU&2k~=;OrGP%n7ep|Y7ZwJ()P^A zI?eR2Revww@57M5=r1h=orw)pTZ_X*KSw3e@G?xNQa`_a*nsuy-!G`}_YrF!J8<9t z4TZ7W#0%TwuZqf5u8a`)`xE8RAeJRyS}8BsF#7X%D;JPiY+mbSYJ5HqT>l1yvi46# zKEm#Fms}8uKdR>8kDc<*K^uZG7Pe*Rd7C=Ej;q!QBA!0 zCpS#m|MmL+bF@a)u`cd^!YKagYfuqD=9Cv4yabb34i>rnSp%FG_JBvmnFWKgJW0Vb zYdAP=0-Sw;D1%{}rQnW^0qtdhT0W=D=v-?CiYd>t=X5RP}_+mkMhE3r!g8yp0wDRH5iz|b$kH<1a}X4 zKgJ|{&??5EVH3R;-Bk0J$iDred;f`83(gsz3ce>-*8zwya)A3Y0fV$U$#mfapJbLA?bdC>Fl z}C=PNP3w}Ow3oT&gZ=C4l*Bhs*$5l%|PsCX_R`bE0!4f^AxL6hT;54L>eMKe8C#ghqVGx3&Nw8MM7nuWy+E%aG<3|$7kz}I1Y8YF%hvRlea z3^ZU`iUz(FZxo|V5N<#SQC=XsdO(Pw*C@sQ(S*Y##Q=Uk!Z|^{Vh6SOh4PuT0D@VArW5sS)pe-o-;ud=_hFPbwUakORpT@w4(Dc>&FgaKXn)I1{ zpSo2?-fFY}MOnkx7X@yXs^S|Vr3V?Pi8LYiA*EJ#!f!GR{sl-Bc7OilQq^;4bw z1=RxzzheB7{?71;wuC4Lhx;r)u4}RX~soiKZAHX=hp8$JooAxGLZeHnzJKo8(&4 z0B$nO3l|df2E}`Cr*STYUY7Z4@+?6tK13r!B(W?m(}N%_2~sPD97|dpLO-M9vmI>N zo;?A!fEn@?Iidl3La)SQKk#x8V)QI19MTuIUB>41ef)T9(8m2y6@qhBV57D~D#YOz zsfW*Be#J(irHKD3^!>v+tOFV`Zq&p*Zqv`SN4|e&;N5;XL=)o*r0+paQG4gkMmMLU z`p>TVbQWhVpm9BmMvxeCG5<$V7&7}<;wgu2jS2p6(8=IV z52d&L17lYADie-knRFGkB+kVQw!?uNz>6IM-IYYL3N{# zgU|Wqk(t=m|DeOr<6~nMtE^9+rsa4swuztGDY+u~yw!@i#p@}X#aH_UHg{AuNe#!D zAco7a#?LVELZ;d&FWfTeYOnweA4PZVT8Fa&Dp3XN!5KIXx2-*L~JnI zXQOo{25zWj^kCkJt?d@7YQLKs8&P5a-N^W(#X1HI(R(iZVL)qM1{vH7Lgzz`NP>s1 z;bavo`E?|^ZHM1QhWwk@6IGV-*-&3k4xK^rWZ!MW1wr>JtQ<6|ht?2-DW*+{P?J3j zcla&P1-yWmwDc?FA)CNL6pX2;q0dDdu-gtGEq+13H!hAkL4bwxjBGf}CNgh1bPwT9 z4wUl6BJ`4^fq5muY=`xx;eNo05sHjZbT~?ftN5eGrGJh5~cXG`CJ_^3UCeOX9FlXoZ+AEvDa0XZ!@1_TzWBcy{uHHQdGDtUVU?<|^Gy zPX$-b$i%c-krBpRQTWCA0kE&LD3Y)2&&QY~#ar4CGlku;loEFtH_xs@%DII(uvI+i z+6!OG7_ukk=jQ0BzbDT8LRc=NKn~~GC8!sIMqfsui(aeV2O}z(v4ov_N64MJ8pt1X z6iNLUJWGAPA6HF=(L%SM)*0AF0WoYQv%O;&{)x%b%w3gc)O)oxVJXw1(O;nA&UMO2 zcdbvJ_ev86z_%q{pg36-8Ob;z=+gt>fQ_N7_41k!dv@;V#UtP`n@ywRoHwFdiE^ z9!QXk_G0+HK3Q`UjXVS@u||)+#~*3T%z0y=$YK`QY*tcSygu)NMllOFv-6etskv^2 zR}EU8^MANDAD}fu|Cn&iYjssLAgU@4F^nO0OwgJW2(nSu%@6v?YQojG3$}#g@E;j) z2=rr%PLYBO8x|Lx(mOCPloA8HyOKvX0A$M-q$1J*qe4-yAb6bGB#}T)3=lD*p^yqH ziJpnpBIy_ANaBG^+oC-l;T(&rsQqNT0_9)_S)l=9mV@-52t9#-8a&H&C|rbSBjbz4 z2z(;KVxNWqdHo3#HDUN8YjfIKiAo{@?hAo{}Jf?6cB1lYfxDIN>lP1k_U04&EM>YuwsikbZ zoul6e*iH01Qu3jf(^2GLHAohvuz8V!lWpfe=g-8Pc%0Z!8GJ<{vmS~P7BEsjLf??_ zIN=NJbw}0Bw4*h`f6*L&5jpo|2X-}vzt(~Gq;6XrTQYr7y$sa+xl>11Tfw6fXa#x{ z$7-cydN?{}wb|n|j_f{&O-qQ!z_cwmTCdn`FHSEZwjgp@z`wdF4%0u4iNp)Kx(fNo zIC8|d6dpNa@J@O-ONPLA)SkXP=JQ4nnaSyDKYo}KeWIdbOXCc{MBbI(02)`Sxfp{nEAn4@S95Vh$eAvp^_h-N+;rK-Pt)IPON&pf?tN^G5f_!Q4dc z1nepU`L(m9m2Dr0+N`!;FZ~I+Nx)$&1i6r7`&x}WU)LR*FmpY9`ZNW*pkLd(VD;Jw zHfRlYJ>^XB-;j|qfc%4~k2nh9Dt0I<)l@0`Yny=7u?Ih0Ssx6#Yl|3%47&fV-hC-d ztX?YyH<(tpoys2SdpDGJJ+i*lh?aYwrA>&h-1u4?phKKi2yF6cM7sc;TAdXL|0P8l z2Becly;^M38Itvg@(u}&tIK_g8<%8kZN1%KDA%I#U{o4kLa_Vy$@h_xME!+rtQ%ad zpjM(^n+phKItq|XM@U>DjsP|+KT1q!>aN5h$O9&roFZe;4MQ{_AYOx&s{=ilG^5A} zrw%Jhh>}&ogn25wn&|N2&@~8Gy-~Y=^iy-Iu0wNikO?>U1RMzP zkf3~X&Gba85i#jNq<#%&q>1~1j#n6i1MI({FWRb?d+zPsh7uL{ra@iC17iJXe7s6M zT$CBCEle_)@ni?Ky<_U~=7&k;O3XFYrJdf5;XxKW?n-nyJA+iPy>W!!72>BcC~(yL zejXh|LlIR1FrFNmh2@XYslxFEOQM4LxzH+5bj?LwD8#-Aqf(JObxhz-q%l{rvgU3l42Ei~+wCIW7#x1_*;o2p`-P zTQJad!nfm?gO^rJ(^u{3g)jG2_Wxx-8u%)i&zw2qh3V6?a0#IiV&8E_{?WvT4IFD#f>4hXu)?u?J?I=s zsEWgEkW_&BN7`Ahtn)jw#hpg<$tMC}0Y<5f9Qn4X`t#aNn!7Lt@Vf5f=GkDb%?f{a z#2pW*3oR>fs-QPcmUDs3eSRKoD-HFnHZ~YGh{OG&mw2rU%7JExnx+-W&q8#Nq(UKr zjPTuqJyM$iy|&8i-%q|Yt)1*(Kz?HBht%tbX7haHd!os}D?9o*5M|>@bSw1F(1E@z zW($}zWr^n@Iy%}GPHB7v0|K%V-3lQ*Ab$xpPyPN~hBd(nLeH$*cag*RNGnQcbCgLC z{3M0t+x*^pdqcJH8LQiADwkP#|57xnB}6`)o}TUop>k4mkb#2vi2im?Qy&=l6Q(-E za|O-AhnzG;N&u$OtB|!r2R7n>FIEAKy4NS?%%*>SAx0_G(QF%5F0K`5n{W~b4WL#f zB%eqN30jJD{2=WrfMCTh59tA0QL6X*jDakPdj_7ZV`xy5E=!uem&ai^<1zryWS|Q7 zL_-0?`QFid{b&C@U9W$-qfirw1ZbVEA|6pZpFcIifu?hFziU+wAFenmH}3;slZ<&_ zKOMNaW()C~L;2W9Lqnsr;o`-M)GL@}a2%Tt$z8BOka$H&Y}8thj_U@xj2_&|soX!% zZlA#sL=nFau>|72!m99a5@8a{B6|TY)cOn~Ig;Mcx+|f+bbtBcO|1T46aj&1PHd=d z1I0jwAAwm29)=E}(`E?wDC)gZiWR^zs9W#c+}ws;IAnoTG~61_)j4+RyVTZyX*AiT z5E`+<%E0c~Pvqd0mAd$uYqYCG1z3$t69S^dBg+vn3HfD*Sq&HLQ-`ck9r%2_aGl7z zmwZ_CW@_ddV> z`}{x8`}%xl#&Elr>-v7r^Ei*=IM1&*Ud){{lOft4^*MnDT))XEIx(h&9}cJ|xKI$# zU{+DTGw%yjr;E34Ex`uAh<8C^ER_qGuD9kO13fm_?c|UBk7|dc4m&m6&uu7c$-qS` z6_Vuj1W5^rAlL+qitf0o)i*U=K|Zbl7w`%?I1#f5u0(RsCZd=Ie$LKsf@7TY66*N6 z{Xhri@b6|QaIkpEq`N=aQs_@7C)i_FnHWRNv>)iV0%5RrpUHA#Y0Iz7_4V(z_rxoR zLJFM2>`uNA4vm|N3;>8~q%1HRo3Dadpf5X&z`J|`z<^3oT#M(xR#WHBU1?&4R!^9f zcZY1B8h#vxPTa<)xDVARbQp?FP=ScoxRZUj^}&k6_R$8CyoYr zWCxpOY!V)Uh8PxejV4l}=H-LO?L$k;6o$Gmm+ri=Fvm~h`Y?%nDb5^fD8k5d&8!aT zYbeanO@awacaMLmNE97nDL>#Ml5`0n5_7O7c<;Qe;sAwrwus(%JU68|aD&_bZ~^JM zqVg#dQrS<$5>{cn9bGgy7O7V`v<1Kq_#0X;NmSF`np=ZC)WKM`u zi)J^-w%EOChn`>1C|RKyw;XriZt+i==7(`!9&MgbdzJ(-1hFI2Jd5r8?zrdnCR_ve!Utqb|dG^(HEHm zxN`-2q~Y_UZjyBdzdpr!K{pm~*KSyU-E)xuRE47>oT@BjnnWUSkMK?&o838cSm z@|pS~f4AL!g*^f8?!PdS#|9;nEn7xLmIvqEfPwsdn!`2hHe!3>-R2~~tTX}j%_H#e z0}k8EHRVv>DU@$80hgjEBCEh z`>~}Z+Aa4Vy@FM_y!aXvYwTGb27PVWd-)cp9vW7+7B5zD z8rBG>zA?N9Sj9Nv(kf=jE{DK;7;lPk#D-C<1E}O-Kn6^}&e&mM-`gY(Q46M)34m;Q zcuwwDaNc~p-{J4^HsbR!ErSMdVz;Ja(lBj&;J&$VR$Tx)gi(%{Q3$b3${(6tLnf@j zKU56D?a2#!2o+F(nPV6j{5CgANFE)!q2Cjaj0~o?DiC{^jJFNRaWwh9^K#3U~%1N z)RquF3u7z!+_gmaFAnxYPtc#hYLn+lIHE+S4oBqXP1rUxP>w|7TO)mzDS#<==zuB(Sva2?|;u(`+s#D|5fg6 zaPHHOp-vB)y0ifkvnXUHn%9AO<{W(o$raovd3eb^4ODf|=thS?BXyKRAfdp31p)_F zUWfH^YElQutWhF}5tc*;WvZyCwDxsdq+o<9s&_IILACoOQKr3jEGAG@x3pZtJr4pT zxT6_=S1FIif8%SrE^gBaRHvaLe>ddPieX;Xp?)D zRoJGam%+HkrjU6<###h6-8)XPJg|LyXlnA;PeswK_$x1Ukc*owpnwHNG}H#G44pgc z$!+3cN0{vyG54VyVlmRB>dFJC zKgbmXy>1bv=rg;~n}cai8w!t5BM)qd#65%02sMX4bkENvkrNl1%yh!?`2gn_3OX>iW1VLSUj;(`00%Br&*y%_ zzpoYiYyq1BafSLGkV|PM1G)B59Mc?eYIj<1q8XKU3-5>|l~+yZf%)UNQP3mYGAZLA zZashUga(T^L5)238!<8Hmj4S;Ymt=HTvTb11&{6fJNDs52ce3mq0Q(7Cdh&}Lv!!X z^n70dhMV(+ED!WmAP%jX=r0D-H7)}jd<01#fd*HUacHmHFdc)Nnbx*CMceK=FPiX+ zN`$T;082HBKAHhf6U9-4J;xl&#@UFF9#N^Co3>**J-HN-6mFs4NyQqW zr4`_v9KJ;7s4YOZGonuoiovI#cxaV}hA#{QorWBKui|_RUdOh-qg2A?`*O5(N5;gB zp=?YJEmgp4k1tKA$%5fhKGX4e2E7-QXQQ_#^Hd`PqIP?Nr9?BWn4^CY@B9h@YVEP2 z<;Y4q%5lGcp$eCI0)1o+`X!m8;_6Ucf~C3`Ra~o$Evq? z@-S$C+O<*+Ho?KRnT$ycrPw&*(#c?TN!Jf;!;%80KMwxZ{i!zb?%51O@(FlR7)LHa zHCow(Cf>7ek+D2g#g7@|dNagr2zr#lvxrJZeTMBfyl1-GLJK+r?R(e) z-aOlV6I5K+kOj^EyZ>O=U7mIn1yn);DzP!JX*fUTlo z5O_g1&K%|m_7xR@{{HjH@`&{;*zpwa8^n)#qCwt7s@d_TaX7*4z5VD5B^8@-MBrH| zAj-(YI!J-izbkejoJ)I5Z}^V>QaaF8?eow&@5#J74}YdzMc zFjM@N)+$0dFF$|aypAgLDx%tchAx3q_Nvcly`$ukke0rM_#stkwJ}{_*(LMmZ)blh zRjx|?kurPsX#L*CPxCh2fPaL?|N&qV*W&38KRq6#B53Z-0l$wd7G`=#-3*HuMsVX5c6e?v!xm?}v7D z>g`#5V*C#uUW^VtfKb86cTH$(0wGPqQZgq^o`CnL0HG7SAT^H9=3s&B<@e&`yFJSrGv}sOIH<{g6<`Wh~aZCh+M>+XZF_Dl78hDE~#QQ2~ zJk@Aae}b@sTIh;a%K++F#u57z@O+Vk#B69xC_~9df9uwfsB3$64}+ZLVa<7Cn)`$8 z`neu3^DJVS4nd`*+6I|yu}7X5+8*B4TZB!lfbkO6%CI9}h9tsr(39SZ2$Y z+juu7CMVy*)<*WUjLO9*l3m{%V!KYvBIRy>lcrP9S7~6jY zWf{G@RvdkB8H26M)iFqnXmld%f$l=gU#9U+npCvldH$Lvk@fuw8yISXpyAeKzlK_X z-T%?sdke4%<0}HxO~^Pz2_NzCEe!m(tqiDT00UD|p*=%{`G)mJwKO_Y)aObMhRxtV zR|28K>9Mj76eCjn`GV{TNbS!NS!(Nq0SQXYm>Db8&M*-ALFJxoK4L>31Lk!9~+3d-Sf*$^s5vgAhkQ zVGg3ZsrX@B+%F8uuJ8?(kYX7<;}w^_Vn!pz-xwOACf)*pg8GpVx0M=KPdjbjtjUMe1<6b4QEez2&{Co*8 z3Xa>L*WD)Y^GD4T_yYve(^p&qt>Mo1;DLNka-pa2xa^#DrC}9{SI0z!^0k*|?Jaep z+t{j3v*MnBS*okXju~?Z`xo}ljYxp85d*kZM0i4gcnW-gM8y+};2HviyWONU&KyC3V8W3rrFprmCXGuQ0)EAQr zEYK2y+K^gnz)!8*_bacngd$9iGa36`%Aa*Hd;~CvDff~-^*{D18nAb}LBbH;zU-Lt}VABc;TF4{@@&?1# zN2l4ms$)&BLkX8CrL>V%3hOiX>r>&3*|YDccmk0C0Ak!P^}B6C`*Mg0-q?9)K(yh} ziMctv5+IGLB}N`|Xgw0J(58@c55lrVQ66k45hzXMOR{!BY@ji86c6yXu{#aU^mYnC zb1pV@!RG{p%7pCN#~$2>-xK%Dl0MVl6EF7e7E(PFkPs6TrWZg+15n~wctb6*$%$n` zLi+=OXN3vDQEZ8CK<1IXooE)v2&21|>y1lR!JEE-cmM#K`1fH_2vHQ?u^=vC8?7{% zw&@^n4&!ck_lFj|goCjuQ*_DJ%TG`|l7$QDehIWRzV*i`6(7X$EVu|y-~LFGR>0mQ zyA3>X2f>06twYKVG(_q{hTVz=SfX(7xEc%pBWk=U2>vfo!^KqQ(WDpDlPF0xf&*Pw zf)B6}6|E~qWC7WNn%LST=0gKoh{a{d5kQIz>f9g{=gap5|3ar?A$n5)B2fTjG$6L) z@h;!Ndw3@ZDR3^;BGmbYUIYY-=g85Zv7L#7LHFT~4jeRh3Af`2XcrpEM=Ku27;2rT z13H+4nD7bKK<$_~`rc@+9{~O}m z{;nTY(}#_`nW8A8aUlXT=Uzj#NGdu$-ra)Rki}aeg`oljRP5gN;$;aYekbTqx2g~7K4>?&BI$?obAGJ%V; z;m=$H+k%G)lHV&tbdO8~>ah@{8oRMu$oACO4VZBmtA>u?!hTXeW{e>^sHg2bW2Hrg zxuLiMbdg|A4XEu+U)Ix;VGD%D}K!0B=437tt@V3hBV5 z*>mseWP)}dMn4I8f;cn@`In4*o>oS5PLYu7O?J;gMU;Z%y{8udNTnhStDU-`uzrK# zTSX7tYs<7DYNu@)h(?QoQ|6UFoau|7F=X+Kl-E4Y9YN*Lp9=}IbhX>E|vvx zMG$AId{3O}x1T)e%C4ByYqc+a`Klu`b%x^Gg9vQsJ{KoQOk-D!O0eJ3ch**GOg>zc~bZimoI^R?{kW~@t#{&jTCSAp*# ztlQPEKJwkE*}rTcop-{@5i~yP+=J0QMdWa*(jJV7Bme#%9o{*qqmn0h)UKtYBO*Nl zRTsGNY%R$gVdt#CQ>G5_lxjfh%f%xi|7bnIWwwv=p1JsCu&%*730yt=P7yBGjfi}M zHWmr=nKESbNmH9v%T_(X%#l*zmK5=Lohi z7|HFW`?2D$XITqN&N$w^mOc_U#RN~nlBMYgI41!G039k&rM|AFVB<^})jiz=~0zboL-r_D{(Nwj`oKncgp+Z8k~~{2}-D z?D<0aD;sCf50b;PLtC_(w z?DWgGj{gHd0EcUEkC^iH<-_38+;_Wq4316M85q@~W2-Z1=O1?qJ1Qp*e!5`gS;MPf z>QvXcfqk55u&H2FAU}m5Y88bU9t`&;WbALKY0ze{(A4xf?JK z>hY3zP_@(x4s3fFN+snSD6hjug<*%=!Yzn7D6fa5sF8J0@YPMp2pen^LmmTgooeMJ zk4_$;es`u7PS)@~7{xH_dm9)fbaFVrYxx^Pej$>dakl6yAQGIK`9LfFrZUE{l>2a; zx7nryNPnNR{jYJ&I|o`^pAD)*V0KshGib>+p1XjG3kR}tt(-Vpr<|sfVq_9`-0)Hl zK?7rIwnlB`Q4ViVFwMIz;LJ8JydXRE+qWl}#n4qf#{Rt`UZ%gT%4ycap@o+;g3ZzP zcL7q&buSn(zF{KKAsnGVhjOev(|XW-`JPdj|Mn6*EKawbhL+a3U^+d@AuaGYs%Y3a zw;^ZM-5u&N8VRgNJK4*`m?uCwjCZl#2ESHcFxdo<#o`04xjRD zQG?4Q)>0ccK*TQ$95m`Hg<*MMs!?Jc>k3~k42$4MW5V#WxEchW+O!#36Nb1w-PqKL zu}GsB*x7_UiX$M>VsYO}$N|9uE%sQ^e^{(obYmInNyf+D@eWCVruBx}F!@p1(pV=S z0lmobq2fy$2dv#it<~E=V^|db`z98n9R|5&G(&ymkAHleNL?oo5ufAokuMxm1GV6! zUGMuGTZ<<|U?rXENWaHRP>ap@Mbvp0%DUtiH|U-i*;R{kkm&zNLGQ_iRoY3@F*D;S zw&Y3J$U>VgT)05e6y^_wsL49~v?aqScJwPa=-m_6p1@-$8oVYqP`%F!(%CS6!D=m*A!t^B7_ zUK6_=V!WuxoP$ShQBc+s0VbC<%!$3fq?XUX}e2JC|bb2xVpl;IIme$|b)DPMUFm`s*KNA7uvrOJa1*_W% z<-%l^I);q@*G!}{)Kz@<|1X>x+0OqXWcXj?&;RbR?$ksMgT?Vl-<$4Hnyr|ejgxnGsiAEY?`i-%@#gPu{<+wm5{SKYWlT@kK#O(y)s!cB#w#8?No#yn5xzv&lv_V7FCF3=J7>ImQuA$XjrZAOWVMOs9z@RD~mpXPt&of6Q>lhYRM0cy=Sh?qfJ_xW;5XDIvWTJb|kqo-n6nSn#&3N{N2hNBsb0}5?} zuFFGpe00H5Ny#}_>}p&O)Zm5T`f>&@_!~n(@KrSlpejc(xOfL~l#YiLj*n-knH4!= zMZ5;gnQ(G2@DyN>x!}LuH-r!UF+X}%%7vxq8;yzU+-px29w>P_6MhrMgp<=BJRBWr zkC|oZpdje9apnnHGTqC`<3Qty0{ZyRS}mw+IMX$yZ$mH3^^(9Zi(D5j?o7wAGG-Bb zj~c0MZaNkn8h@@WD9PG6p*+9D32NanlQ)+lS%1C=`0ebb?VA?&X17d9n^d@Sm3Q3& z(dbf13#F*uq^OpD%fRPr18pCd_7^CY9lAK*94CMqzMQcyJGJY*uSMPladz7+{{H3Y zSHDs|`uv`jpz+6K8AeQg^JuZH8X6xZ@;&!zdajB+^kxOaIIa&)N|JKueOJ}?Fs;ea<4e`JP-<%s6VxMi(QUWI}q8}XnJ zeACyofuAm7f?@_!@MLj@bLZkej+XZ#B331TjZw5L)AtfieF&98viEDOuO0V6`$84Q z;ZfYLAIeHbj<0^SKCd;_?bVYp*DFZ0IB8w(WX->H;75^Y)lof_58k60M&9jTF3YPw z$7HA~7xNdU9Gk@|47e*d7SHB$)n=T+?sgs~h(||~XXpjuq_}U1{?Ac;+h#l!_bdOi zkgIe~#PKdP72@Qmh|4`{Bt=hK~$9B5NtIhGX~ zAc;R0n=|s9ef*2WFb5#RVW4YSO}MICl=eiI!qwS$oy4C6<&0e2 zCxXl9J>cvA#pU`B_riVFeCBZh+!sGPANCya*=HSAYwU3E-*4*Nxj}1JuH>Y4t(H>g zRnc^fNV1|G!2~s_3~!nL#=G}#3m*MsUwO87#`Q&km!lJxHWr0Hl4*L{=biB!m+^qM zoBP>%JZrvQk@WIEeXwGA^YfW2Yt3}bvxGX*#iD(zGx#1B{rf$R_M>Y?U;mTk^dGLz zKi2Bt+t)mLv>?{7e{Iga5V6Lh*q^iUx<*o;=TG05Gmdc_YlJ=ZsX)M6;=!lsO^OO$ zdeMI=gyA8M35p|&(j7IbVzb`j2G5I65ts7%{J_lY7Q`(heDYQx{c}mLF{mlfO5rrq_=xogNK!tSqIEg_o~U@-31*S_>nGW zUbsuV&S{G5on%YM+WIwD#FbE_SY)qF*Mq@l%+g=*NFP<}sp660;Tbrjs}03@L#UU| z-DFvefuOT$+2-lY{A(#!#=@6hFU2#5!M|U~+9F#+?@vLmY6LgELGL;8w#~ zFw$%hC`$C&4&7P0F;BETo4fF}wx476L1XHc5D_Cx#Ly#nd{x)g~bR%-PA2v)^M z$@zNsUQ{OLyrHop-`&&R_DOAgsgAF+v-4$sIaxHwoa!ty4D9QCn3mHR78no%x9qgd ze8KNGPA#&<(vx2l?RVzG`B@y6adfI>WxPZG7u`m9uIP_|-c>`f?bF36{hjZ0G=9aeP1F|g^0uUQ>2N||v^ZUg5ax!)h!oQb zXlSeq963}O^8t@@d=@?*|6y}wSmxUiK#JOeMh@p|8iip9db)6eA@=J~OPi{@?zG~m>tk3I{&wismbkffxf*LU(e?^x~rWgS1m z%-y|x^Jb2gLw|?lKwo=%k*abijL4Gh*`iL(B z6PMBNG~L#fYKiXCI;r4ffkO(iFvLVtVk&--&)MPfsL^AejyDw2+DyUd<3)rmf=s09 zZyXVUT8YtXxAz$1M~=}^o952dIolL3gO^6IDxDio+1kh#E*{;= zu3xwyOx=QP4i@*HZg?CM=~_-vD9$7@=H?;XE+D0C9F|~YoXXRq__UVQ|S~ zkJjPGyEyQthBs@TK7Bg%G%O8y(k&@&*6sClu^Yj6xEPMXOZe!ztP{fsd!gvZJ;zdw z=>xQG@G34YE;}BZV_xrVXi&U&pGo*7JXHPC)#!BV-Y+|#4CV=^oFa}6wO8vsh#Cx!nm+Y(ZwH4?3z9W1=o6)QQ(Te&MDJ# z+ohuOUS#TfL_aP2?z%)A*(++@RvQtMxw6diorwh&wqatO=K;Nb7z-`1W$G{tBM)bZ z6~apMHR(%~Warx>h3K~#+wy>Q7VpXN8l;lq{_YSrrA+MXK+Ir`-_N}vM%oj?#~+W$ zSW2&|`PJU4^7Ot=818TZBb&at`4Cr%E}IdV7m(g zf4WayvQ_W=40WUL3(6g%wB5?fgH_{%J+$4-2oL0!G$l3!C~R}=97&;$Gtbt&aIk61 zSZ0ARtB<&Gx)he~8`@Tt>ja}79ma=zn1Wkz<#8U`1Y%2d&gx@qC%r{;#2`cPfY!JM z79|>n`UvtmdU;qK{+MVXbuTLESQG*RH15^n@6Q}jhmLh?W;!^`u>^!bB@X} zZ^5BHdMFaTF}VZBrQScd5$WMusH_9KlWy@@Rcv@15sJLJW zhv|*GdwA64O1ETJKXocha~SxF@SoV*oUFQMk1Nvbx;zA~Tl5wf7nlQH*dkfvg@*o7``2oInAe=D0#OC~6FWysABPsj0R<2*FF<)Jh;hqLy`D?RkJAxZw^ zK;ba-*7wXnmaMw7)G>X**;U9gN}N8FQA+yyb&HKxu3Y)~idk|~LQzSHftR)=8&;z) zIgyF=KBu?oytnxhiK-2+wd(!eecB!vp`?dyz!gIMk8DpO6x|LEUQ9)m?m$l|E5i(h z8|KCaWi(6UWBS>)Ww;bg{8<<{(AV%arJWm4RSz8Sg~0~Kh!u(73l!VqqwR!f`*Swn zHSz;y6y&!ejI!7z2zW`xc-FK6yx*^}d9om%l?kk{!i~4@{~euX4jet;7M+kc2Tumr zd=6)2lpHXttnLFV|2gtN0X^T7k3vzF_i<~eW4Qa3_+wXmhIMFKuyjj_F~51_s^rpj z>()trytHIyvH{)^eW!vq*y1qI>n(Nq7}k^PwguKK!Y(MMMKIrOYYSqjV`miNY}fm< z!fbd0ypoJTBWJ)es3YDCRKTgln01|xy(vFFy0AHs3FRxP^Yzzde%G&vzn9=MeROE- zlm&h$M@?KeOmta@CqLdQI!kBZRFptjtOSS)vsOdww!mbsQ9FG--Q8{5YA)s1i(6@$ zn`A7Lvk05vKOWJ4tk7QRA8}OF(jHo!^eLF)5*AIET>lDw;8a?>eV_NG-OYORa^`_r zWrrSBjvhLbDH`7Dls`0r|Cfi{JcM$6Cnp8-;ZBz>z^Ihtk3L*DU4v^Z$L@dj(ks68 z&MVB#q7amb4+Vqs4VjK2)x-}M_+lCY&em5uO;LhBxiSE)y-V3rz*yCWmT2*1i{Qd+ zZKCWmvqjE$oLS1ZtvblSOQYwn`VQ5v{?4tdnG}LIPucKYwDdi45S!yPS&F!Vj`{>F zXhK);!DQnBMR&MMjeYZg&5s`S~XV=cgrc{SH~ z)i=AlckgmoX+Jxn3o&L*>vZv6YXmbz4urLh!E|GXYm1oBHto9a9G$N8UYX={qRpC)P~Amd9j2jE32rh zdxFPX*JB^%Z>dRnp|9C0?Lf$fYB`XFtJ%1CzG4z-_nW^VHHjVLas^g(3Kge_dO%q5 zL1Yo6Rly;$KvR3qM#b&n%&b6^pyNMYtK}rF83)(d{+-0}=#zVSe&rN!DBzRYBll zCE{2V5uCsD72rwt)5Ko_>#+2>JGu|`f)eb+a#sDw{e#HCsePjEz6U2NIn5e=M4+6p z!-{?U8!nW*ugZT5Xc?>RIs8HR2l&BzrP=rMS0{hv1mZ)`mHZqP8JjD`<}3?16Z|w! z>x|u@wfgKXEF#9G6t{@Vf<#0KO1H<6t0kKs)qJk!xA}DQdrOtv#d+r2Z#7dipJ)Dh z5g|1t$vXEuZC%)jt$v5~K;OxDW=uRu{)TU;Ey(-o%*cXh~~ z9el5WCa{HO&)BE+rn&;Kj~hIyYW|$K`gMDrTp*_0J!<}bF%O1f4vu5QX(7^}n@msS zY~#0cXQ5~>{Jo`nmCIS7mmif(@h%;s?-m4)77N1))4#T%^EDQL7a;wbmXvm-%8E*> zDu2kR8`v=jcBR^Ag3XCVFX}I^pCai%hwgyK zG$o0frgmV6?FV|R!Y~FRu{~Fs`%vJ)T~wiBJpAy0uTh@1z2Y;|17alvDQaH(`gnY?x=V#27tk;+UAt#%^mqH2c85hFOf(^|}5BT0{|wUlDrK zFiotBL4FtxNGLWjhEopeBpu~sfeN<)!=$6d)0&gM7~uBjsVZ;qKrhqdH(du1QQHBX zB`MobSO@Z7L8(q*mUi4*D^=6`PsehZS4>x1LqJ~^Fstv2?m*^J-{UcFdsk&F!1_D5 z2SXlKHr4ot6b2!=d$8m;2KQ2ehp0a<9rtmGFp2NiTp4hzYjNXxBeyf z^V>CVIS2cz5+5(ww)l@_yq8>*U3+et(rY4&Saa4N-|xB>@zl{kO{eGzmh?y34G-Vj7+1!lS_*E?gF>K; zsv%`@@Y7l1hDvR|Rkk9sZLdS_-X*r>Pst2LupGte zuBd6sGHWx^m#x=2^+uu0Ua~kf4=l8u@lu#FuL*nC!%WQ1+kSqHcyshQ8IqB z2S0M0sjhhn|K)x;4ye8UmG_!#kI@3V?{9?TEM0^WUe7#m{7v>>_`PFEEuugs@~fL}|L1+-ckUQH z)oT^-@LZZ|Gs8dFZpDgSo}c%gLXZgZ-hlfy2WJ}{Zb-&^z$IOU;$COs?LRFy{qvbA zPJaa;;LSaaZGvE~No7jRP5OxHN2n}9)x5Y=amNlwbPZ_e5)2KEY@<`<82M={LT=K~ zwMKA+Qy&s3^mtUs{v%g{m*87wO#5E?5q$*}uc$qD0nknYDX;bSrb;Z4m}2z<>rO7- zzlq=WdPLgKWfkzwzZ$ zM+7fKuSXPvu>4oOd5@%Z)F`OqPB>e~F8=3@`n?C-vTfr@EHSdMM@rHI^P?DNg^UA9 z@5;YKc$6B(7*FV~+xKynx7bk?!afu#*&G-NWvr^~tO)N>#F;T;Mn|XVgO@WEw$P{4 z%*X7-{^N*^-7kghUC~=NX|C9Ev~J`8EwqD`U4q-4W8iDziRkL!nra;JU@OmG6c=x+ zZ{PeJo7^?<=l3vrO;gmE_~Se8`#=nP;o)+a|8s_ZIvmkYrq8$d93FNXIM5p4Ew)6{ zxHzY{sHpbsk!v}Sa6JhC174o`gH5kcucG;47e69OAM}PC6cjPrMK^Q40(=qSZ4R!W z9##qAg^Tbxc863!jI&0gI%VG?V}tD+Qkwf3Gpu2Zh(uTp%XP;R*XKo7I!rNhlLckGlD@r0!kjy0-4$P!K z`FUrU)?X!K_kp;1oSl*Mx=%1Vg@gJp^1UV=71Y0f|CAq1W)8k+VGtH^pmz^i0hEnC z&Om=x@C}b~oC!|*xsr5~JMa3p3(>QFD53n=Jz{opNgOAxhsxCiygpiRmJst`2@ZSc z@&a+)D!>7UGT&ZIqgZB+z*!5PDidAcT)}?af-?s>waEX02QZ?DVw;WYI)cfjsU0OC$7XN?AV+_!a*j<)quP!e| zA5&m@LH>1HtsAoe*@#=i`9e=BGguI z_PEn1)e*FJfs|ZQZ3eDG=Abs-?9!wBX z6elqy8B?vtLG#v9kp4#N-yU?5>8^DfBS34^`8(AH+vc6zf=c9bR1Rrtln?4|9J!i- z3+4qX|54C6JW}3bC1Mm7l_4+zkzBb5vqXq=)FazhB1H9AW7D;vBAhjKD*j-3KK}N` zt*cTifdUf2yii0d_67gkuZrRo1(PhsBn)pAq+IZ;x;nsI$K&2?C0vB+Bk8pdSb}Bj z1=u+=mB#5lOR?rpQ#Jnp-et_VW8;q13q)7-Whh@UM2a$b!w>eCiV*rZkGpfsZbOfI z-v@o#3vFzM7)?kEWbLAxkt4!*(Di*vm;V!PHpt!i_}dlbSE^x4dBr~EpR3Q?xz+x& zcZ>A8@UNQtkz9kUZQqnXh3@8$F_-!{TF5NiD8Dm&L^uLifgH4pLuCLP1$)eEA1Z0s z;7#6`KgA_-RNu%Y>WNA7*sIRwkP_#jO|>D54M26^?^r=6t)7nd!3j+JhOq!H8D85C zE8HBZ%K#oK7u&3l#J-@uR7E8Ru$ORYPz$jH%w=VON4%l;1}5~xS`M7Af~4|!Voci1 zJX&I6X~?7)?(W)R*K*bxb$=`(K`p?mV%xWet0B?q8sJXv;>?5GOXZ*QCGd&Nbm+Pp z30^egk@HR$?V+{7(L>;%+KD_p%h>Q1sp}D751QjK>olFlE`j1CGxFhkO{*r9N_TqO zDQBZ*kG^Dthsi(9%*wJrWZ+8BW=Kj&UkKH3t|=dX^eNaS2H|eT3|eGH2Iv2~02Ysq z8->y#7nQX4KyTVW7M86CQs)M&PYq;Gi`R=T>LHk|k|Ok;e|dtj(SD;LRqoAN{K!%L z4k9*TQxfrlZ{bIbP(<7(#zoZOXJ22vcp8`Y8vI3MJ{gAi`>MxrH}O@%Z(>gBW%(ya z74bh^lCfvod=D&~A?T%mAtN5dWD#F&|Fb>N5XUztY@*Dmb;E#hAVh!e?v~~y2vg%$$%nphGeV?|; zWsc*w-TjaY*kh0)>*5NPvd|CQmSbiwih@>n%{u&fzE)#-@MA2NN?m$SIo|nd6QPl}J{!shPigK@l zHocck&Ps^&Vh@l!?nQZj0Gq;PQ}G#=fv2!1I-4s#RR1xKu6aF2adig7g}o;Z8$BNh z;YO|(F5o`PfrC?mfK3`v={iBeHpWqj2J}9_u~vX|qZ_2T_;Xh?ow!HxlwOp#lmZDVrK2)%$EKqxB@(pJ0hI&ml}es-N@Ieu7Dj1ezI zd;7*jx)WGB+M)t<;hYN=^YYf!)rH@^tNAe%KahW?uA!0mBHjs*ZK)D*xN&2J3WQF) znqx>-0HePS(ACo2*i~Vn{Dz{7qx!3TRG8A&|HO$iWEqNBRd&#cd@gK_g46g0&l=P? z;xowq%F7ECLZ3*wdA6-jB|%ravCtG_PDzn`EW~>>kI%Q2vQi4Ljp2Hj8uY5&xE>~- z{I%)x+)#^vz*wC@+!Tx0EP|E;J<$W1B3Y|cPi!(W{Cnkg?o%S`1y{F3HCL>ij8z^} zs3ctnp6OUKCLdyDRNyp8F40J2ZmE(~W5`#82}<-6b`q|i9$v;b7ZuTURyV{PpI&wR z?8&~t^^||zz@hF17x;F!2M+i(CA?crxE|xbIV?0L>>xcPuLAov%MgIttoa81SbifB z1{Gw!1PSMZ`3wdSpI;NZU0dVEFBffjA6PXM)q>E}8F;+W#+>*0_fZt;QN<0p#~mf- zz4sLa^YPQKqFtELu?jvPOVOq0w(h^baE^4R+c%pp0{Z$d>yA!2Hj`B-*OBd|BfO#` zn}?~s_Z4`^{wI`+dXh&0vJaRwCs*}1H#Ec1*&7SqXq~E*(ZzpU#tRp)oM?+Df|@2S zf%rwzHi6E4(3gB0#8hD7Kzseo-Td}90B-*2IUHD%*shvqTNTdMFMakApXbU$iZukM zNlZi&lN7O}0cSq`wKGt51K;uc8BC@N7ck zU*WGIhlVpRcg>i>Fwu?c96;6r*fYYTqH?9aM7SY-iGyQYB}M!$H~;?lbnBWq0&u-B ztXqpMP{W#%EYGFhFz5*{+)sI)=BEU@w^{8v&h@qEkiYuJ3onA2R+_SlJ zCj;L1fvIC3J$m$UyMK!w$fNBXV%xjZfCeg%G#jzl>DJHWLW||EVZh*8(CB7DtMMn2 zfZSs50bz-4AnB`(s0yNP@pfFUa%&wYA|&89A>soaO<%(#0}u2Fw2%_T(nAac~_` zh&*~=VU{mminqD@I#!Gny%_QRz&nd6TUS>6wJhKUi8PdzaGh3hZMpedJ!YOPq=1Kj zO}?R2gA3H>&DzDHte7X~>3Q3Lo3|51VdVtQl3WybK4p2$N%l4eGL$z*pG_J^c;=Yj7)1ob4PkpcsanW%3QcePh>R}HZsO=#q6yl#4 z8Ff%&#~}U{`=*iX3i1?w^RVV}QuQ!@hiet0qO=)zB$Bf`RU%OHvrQU}*b zMs^Rf$CD+B{B!GvwGF@g>nvj4$nXeGe6dx<$!$8;bsp~S?jZ=SL~5JEa~F22<}0nd z%5qe`@aN^#QY!$?rf|gs$uNk2&@Z!y69AnjO0syyHQSU@6-0{#Y^*K(Ro7&2 zNjmDIVP+wg0BAT5*2&#ncQ3(>DU5mn22Q)Sd%FEA<$B&+`&{qIwI9VF0f$WrnAl89=a%0V0#}*rXec84A36lq*J^@ zwETU@GID>PlDuy#Tu|J~cN_nVd3WaD7}(SHI$znoo=jNsg+8K&N;Q^SD?2!={-u#3D1X}AYC*H+3i5d9fkFLiUnqRsi5Y?r6mU51?}Y7) z{4m6CE%iJcGd6C2Gcb_#4l6{wRLf%5d`FWTt*(j=Q=QBhGy-1nNztD&Dl_4H%0q0~497DMua zcjxkcuGMGouN`s5uQ(A`s65CSMGi+vz&0*%3>HNl4pUu$?jIe$!5il-ZDp$M_RPTt zW3D*LKGQw*@(hs(4dMECa{VnuSc2`qIx?*k09vGMBvgk z=n5r~+K0~1;IPOJBW& z!8DSWQGCb?0$rgGF2y_;V*-c=yfg&egzdk%%(HxZ$okvMS-u^zR2b1^sXISQ=QYf( z6vIeB7q2+6`6tR@uF_0s4ez?7NfM5a*ldxkTD;XPdS)p4AqffV2{@35%E};p6b=Ng zO=r;9M0z29)gl5CVX-M!m^4QMF(gsG0mc5Yl3w!eLC(}7n-L-K0IvU5fRzQ8D1FDB1)trXoZqbBj%obt_vDQNtZ|5pJm3Y zHC5yD7UvWx{5oC~mgG-b#+vbJZa7{;1Ocx{(vB>LPzlk@7AoJnz- z0F0Zc6GCBz`KIUOAqI7f!XS1h_YUDCliaz+5w2MKt719<=W_vpwd8mF80D!W<-IF=kA*nwYdT-r5ED zgG&f8W%mK!?A{{L=vJVACW!ra_BY!6oCq8w1R2Wyl%~mH2F~OJKvN)fO%Qia)ym=} z&kTKKH)t4mbIC#_{-ZkWx0i|A{VP|P3hPmKWsAKR7T!`=;a&@PX>qQsFJDei{~pd8 z0udTVLp_|Y{%Yd#PnDE*2D#KwPpes(&;sS)K98FtX0&^K<*7m$BrJtNH#K>}>d5B~ zU)txWT>}GEa~+^Lm=~~@)dBvI!$M7s%L>gQ_z3sC?Uw^pnGgZZ`@W?&0H59wPXh-T z2G_MUej;k$_hX3m_W^f0S?W^|k}kOc3C@4cnSwNqOiJaQ*6vNX#zBd&*{wklB!g&7 ztRT^bx4}2KL}>iD`xm*FLTl(=kLd+)zK+QP4MeiZXQEq_P~ zf2z=3&It$7L~+YvqasTN(nPI-2n_CAaSQ!kb80*V(jYtvPq~bZgB$!X zEG$u>SzM}M23O4_%}`zQXecWK98~<df6azK-U=7P+|6$44oG=;91f<1$wENmRmDS{M1s4+BjSMS!#$VRmsa}=s`NVbMGBeGHekEoDmHle+1(0}3 z;hc<0l-EoW%5HzUgtsLxx-#PCA~rr%8#Fa(68{d=+EWID9mD9DkV3{$;t<-;geUW0yMfAx^fgB(YzkYE>kKv#j=Mcpssxy$#BjO5n zVC$@q`aMqjZ-${sdj8`O7%He2f^~NM)yOrW=*+e2`A$YnUuc{4C{*(<@R{BK!9Hs~3ARQ{ zk$_TAnMi&La-xnARFbl^P1*r58sK%w14eeG$^v6wvNn#J?~bQg|5=BMQ7-BNZFbLi z)sIzR_mw%T0Ihg)g;Rba!-^ft`v?PS+S9;ar!BQ@)}a>@B>xImRpt=QN!*$ULU9W4 z8nqO}fCkV*{Y&1(eJ51#7eR^cvrZP$)gfc4k8zYYI?mPu$DFo6jyL}a>eLYFB@2Fk z3)sFI(0{i-xR=B-k~2_4`(iitD>i9YBumT+Xx*)LpQPPzQTP(O?$Kz1s>;-V1dHW0LN+r_VsN#OCi11O?B4K!e?Kw z=#CZzyc`7aINf9*uTqi6AWsQ|m)o~*>Ahpp5`}K~Uq)Wn?LCuLF^ZAri@R@mAE{%V zL={d3jX+&z{jWcer(7qW>(h0E-d)-yctMg# z*c|+Y^ep=xqtt;GaC193)N^;bbg(SFe+GGUUla;=e25K*SNV(ExwJAkDhiY&IPv_7 zr0iG^uf=9;E$Y&h$%6^sey^eRIvp+1clygNP>R`!6xcy zBGjxpLfpWVZsf6GtLXgvcIz$b^OzzP#k!1Tf!XlX`cw0aY1JdW) zps(}W&?;Qv6{2{zKl>XekHtB68^2IgrMwa|-Dp z3P(FHT!3gNi@}iEANw)K(j;cp#7G2tGPK2@-*>%)E_8Xa_E1L^H%x#_V#D$rgIr^P z@L?Q7m6g0XP$qFdiYaM8cDrs6$6+MolgfqMIR)txv zy)O_^9JJ_Ph8^VCZ+dRX#T;Z;L1B_X&>NRcp@hOLku4Lpb6pU0r~*oOC81&X+tB}m#UJ(W8BHMv5jBn(Y07Et}eAPOTa z*y0S{>*&?G`!t@4)2s_l2}=KrcrgAtkO_mzK60!qUMyd@d<7fr;MBLT!coIFVB_<% zn*r%q8Pxd&20yG?EI;>e9uH2X22`f3G9?Nfi>h% zMFoYIi;W>u)lt{bn2>YA2YbE)I4i$GG+dQ>iiT7q7P=jf?a1!3Sr`5tRaH=ABTNhM z3wYwV2np%tM@O#8w7vfRwM>+26vpsiGsJe(ng3cUE@Fdlq6kw5{`)l*2rr{BFL*y~ zF3*W$b+?){rKi<8*z6R&x2t|vG}n+5f_R9@B|^k^QxJU_l{_Tc#A~fF`hrUI98E58 zAt(=<7B%OJVw0%t+h>S{r^LPI$*7|rp5S_njL4HlQU_`ZQmFFA@*N+#!;~Z@hEdI- zxd}M~jD!XH=dY6?h_VAt@7ueD>Wv_e9z=P6dIHoASx`p0FyA-)({}t$exjhz$J>K^ z4?GlHA=go1Rm6OnNirns(mVa|MPQ|%;Qw~FVVhlp1W%6B;omi3v#^6L@!U+)e!MN} z)g>x`ny#p0ITo#|ix1PUBa4X4CFegmV5vJ|I@d+ITpqpqsYB7!fEobFg-XqaRP3VV zZ>GjsykG+W$?Amyc%t3z6#mE;O1Eo>#$?mkUgUQvS6rRQ;xmcc!?OY_%duV!oH|8B z7?JLn%Ai4*YFFNFJ@`D!f$C7$nUTz|@8i zwRDY4FhtgO9gze1Jz4`C)3^U~hX&<*db_r}hTlc$&0}|~oe6zM{(X=J6_`}{E5q&_WQ)h-TlQl{oEfH+k#0dKv19Jgsr(eWk?{JIj?{RsFO?ZJ`y=dGKjP zZ803C0aZ$wTyn{njPF90Q6K;*P*N7-PS-)Aeg9`h(EQ!QlScj*Yi|Ni<=VCnuV#_b zfDB1xC}XB%C=xO!Nl3_0hKfu@rjm#xb3{anBpD-9lQI=5k(mq$Wh_#B$F;QEd%y4V z{=fJ8{lE45J^R_a)>`+yuKT*q;W&@;ILQLlP`3f;OsVnwmQA?d2G~3SR~FJwe@g03 z@Y%`owfLdQ@b%{N!J&Ild_6lj6m&hAjL)XP--1+J3aZbCQ_MlMcP1`tO$X7j$pkQO?Tq}gKbHGyKVb-SQu)N znAD~b*6e`6IXh<)0=~)3j>(Y(l&QcNOnSI%34-#35Rj zv1`9iexu7Ecsj4a=D`i%M-UBb%*p`G$Ml99ybaFT|yoox*CmQ^% z{1df6Li(n@hU5Ps_@fnAGMnY8}GzioP6+tLkPlq>DxEVY~8&e{b1kgifbZ7J!bTEMZ-&~dW?xHN$60u$y7YM zVtWULp1wBx4}?hQHoz>z&|)+BbifWig&^N{FBEGPD=R|@b&E*g{1JuXaRFzECbeN==;*jed}C-BC@$`P0~Ony1junjp2(|8usUiJ23j*{ zUy;Il&T>MTCXptw(8#$f9wPXqwumAoud5>~cT~r)`&B#e69VNMz|N5`J4R{~NIK&Y zQ)-Bjy>t~xEVUbq+%W;-ane1|rIJaME9RU@5I+cu93YQS9q#gV?DP%4rtQE+8r$c2Z0GG5xGoMYx=2~$k0hJqW zcL^W8kTd$5?k_OzYL6BHAfgs_n%if#A+shbA)}-bl~gleF9NPC8S?o9mI%J7)dpTB zH#=2B`cvV?GH#(E3kTGR5DZB7h9DvMQmo2A(~c&gE|q0MSebBvMp|;e#@RsxN5(lv zLeM6BGr~9|;5N4^2F_nYuUP4q9Z>*GCaa54zMy5s_tm}_!s#zSm<$6NO*|sOV2C&g zK`0Q`|Eey=Trd^19@24>CvT``t=?jRX*JGXCCHsfY<3%OGxdPl6)DJ{M!hM z2$D%?8Am~7JUrNjh_es;h*$rms7{E4*;K4W{%kT;Kg{YOSwUhaLk5Ug8X@VRFHizP z22VDQ5le_Y?uRN^@|PyGTM_+;9-BD|w;~PY7Ky?(|xN});&gnU=xZ(9}2$G2$fEtRi zght`!NMLs=SWf?C2p*)^S^d$mQ$H|BrDFow_AN*z#21pTxe`92d{zB_)@7@8B4P=| zrX$lZMqj<$I|()}Lg*xXSS~`z@tI~eMtt*P6&MYF>N6d)G&e<#R;!?LDaW!mF2-rd zP(Rk?$js&LwtuN=Zi%)#!u-kzH^`v#0 zw0ik8FrRi<;BNf~{tRcL=2!_5XHdn@LR`{=lIUv^-Vk(|{yu2}M#ClV-jnv82mn<> zNfYh9vsi?L;<<1y=PXo}tD!ML*0_h97FFK3oW@+jC!(^Y2^`p#VN&|9M^<~ENC2Ql z;Y_j%(vu8A)J*jRX;la%Fo&_l7m^+!?!OL8K$MXFebqyxTMk&zb;xMQz-8GIqJ-C% z8(&eBfL_yXQh)5l-&LV7I!87nF(sEO5jaH9@K=eM+AZhpG9c4Fr9~=X!0U+fBkqWK zipoCPLAqWEXvako4Z>a}OjZNOR}y1L;csKZ6Zs^zViFTK6+&zd|4&(Ci3gw>ioURJ z^u=4-ze2>yT<{GpU5->GQQwUu_zk+zLHj8u8H*8U3I?aj80Yj}{s=1$ppUr%=UL8C z6Dg5TgCB06lvU9A@*0I2(g}VCb}N3$Ks}3k2mDlAK)x^+#Ak^gVx$(tG*^AqM2j7{ zH}aYZ`-i^aNaa$i%UW@vrN7dNlWN;C)zto(*d6cNNq^yoU?*5z4hXRLT1?F$^tjT# z9R9CeAH$WT#6({I)=d0|$laZyPn6+^ZH`ApEy{U6+TK^iAWPEt<#lv)_|A_nMVOG7K$*`crzn>6m^9~WW{ zoX`{ntFq{=jp`-R8A8$g+l(TdTuVCYN`Cy4hB?hyF!3)Mru_B4pDEM9-*@rv=QU>_ zA0I?T`Gk$F4au#c#!y%ne-KTqurEA>dnmtu?)c+vuZm(cye!?6OLs_P{3~dPAD43$ zBj!Y>Myp~!E@7oMw}z9I(eJGdm5U|#te(qfx}GW-YIHQTiXB8R;eu7AA4FUc@~Q_y79L{ui6zyd%`weZ?P?tYa>G z!SDTdT=PHt_J0bOo_KWg2xw*nGd~8hQ9MmXqXpQhTJU{HdWvJ6-y4ux3{&BEVp)j4 z23RSD>N08ZytsT_3~=P)x$+WO?m#|aX?e#-C}NJ0gfP%(J(3U7`i4xEhiK5~8CXWR zjS&clbl{=Jm-?j%#oRHX#Y6aoW4PE+ByK0iNzg}L2}e%+xNwo=b4Vr0ZIYHY5=goN z^db=WZO|U5{fS+dc!jdQlIl9?@il=?235ueg$B~`L_h+N?~2)-YImWLf9}gER{Ga> z-t2^jI^0c^*hsyfuyg<&n2_#pw=sfN15)Er;*i3i1zsig9L#Y0=U zDKuy~kt1Ps=kyXdgkczIhwUDYOmjT#>Y%PT-K!VE(8uEW&u>D>l z@4SRo332!oUed=xm>@_98&NFFncZK-J7`Tz2#jdl2TA+xf8cM!CFgAZHxPpYuoBme~mfT!m!)-R! z8_@hh^umdF+0YXV=03t?M@+&zHI;)y#|s*QIw%xj1QG&`J)7i}Zr=u@aTqM{yQwreDlZ73&&KbD zRK6nl0{Lf6<4fs@nB#ruadI~mf&|h{LO@iqn(mP(k-WT>a{E`4st|d3XYl-gQ(vAl z;6%f8-9h4W)75@(93*uLiuk%ekBP+N=+!WCKSk)tqsuG24*$2N)c<{W^8cnq@!zz9 zZS@?UE{(tJw|s)m)A7(&e%CHI>F?U4Knq9trzsNx-|0ja4CR=t|L9aHIg3!NNKH>Y zK-SKxM_j!2v%ob;$uMJ2q4gyPLJ;}g%E`&u7k?V4{%z=aUV1}9IumD!F&oie5&bN` zzFCoT>8*!UGF|V)9mdhF=C^DU^nNA$r%ybR$9v_ki>ANLx$I`E`^NtncE62!u09l+`pJ5fjt&S41m;mS6J6#0Q`otSNWk*ca4)8>pQ< zX!*Ez`WLi!TnW@ch&pq&zbP9cQTP@_`7#GEWyRJqn>7~ud!6XJm(P57Ux&XOx~Hyj7D))AOkaQxb~oy z2;>olIu27Ms#L8hJBAMs))ZKV)e(N<4D*1IZ?fCmoH68pazvZgQGESbS60i5H=O(PSmW~+T253Y)g9e$2CEL2< zeQH&uKd+y(?QO8Awlt5~fVQ>a13+7tl7Q92NLq*1z$2-Fv=-)03M^$n8C3y85q>M0 zxh3Ri2h}`??8h*<1(yB_S&l z=mgCw56XWclsNeCzQ|dt>6p=~HJ5~Sajg@<7Z*>m3l~Y8$aIWjs;)UvZSVAYTy4pOzy;B>I2=%geV8LdD6^yn?lRa&8c_nPv0dLf53pwY z&4_Wf|C}%4=O!1X%OAo|MB~c*ViQ_y$Z*x1<)P_ro~C8#qC*>vo^M& z(HT@R>YN19G6ktYZNf=I6*IrB1tqa?U=p{06cXp3^sUyD`VeO>zta0~{L=TCW9CYC zT*n+*(ODPd2StRVp05YAnFRE&5)zf+9cw~sGbnO6FRc2PHo0&@QPIYphy#Jf@sbZ` z^0V%qZ2ei_DB9oCH`;lKkww%(1)vaH#1mVR6&d=GVgNyMN25xAauZU(^6eeoToqs% zj1_$v50(rM$;3{LQGaiA+kgNtj4f(9};~lgCeA1PF}{%qoq3dZ^sz zQ&XXLX#e&8>^r=1F^f&HKDOdvLO~gjz6Ji{z zCKc7Iy-&Y(tLdfhE+~MJpg#NSIbpNE62$Mhkqt`X6V|JnZ{!WF+7TQxCBfqO;hJzU zzQD?DF5OI^hco)sa&Wi!t=u9Zvzxc2PnfC3!>81`%?a##3#V=|3BKCo{&RO+#t&=! zIQh5G{wFUJo&BX8H*wx9PsKb|IMI^>Xd~g&T%PrKmCfA-SPRO7&DcgXbd{B3Ej-Qd z*xpvF@rQDpr7|Do&owm-tn7`7=&qYJ%xqg56G!Uv^urG`YH7yq%5>N8{<`;R?U1_b z9gFMN14+L&#hLmE-Kah8CCcRA|21`h3KsD3$H~dr z2Dw59y-Rt4&i*eZX-5x;3A>JcgP8wdWVqg%D5^ibmNg~u!Kqrt`QuAT!wqzm9>M`f zZ&OE7{CvePuFw-}F!fDXE0rx??q^%I&{reVL>JSUYKv}3lsEq$cTGeY7ZG5OG|S7C#s$w^?*NPvyqxl?U2E|GS^ozcH2Nyx_UwE7H6X5eH1%+7XmHj9DI&ph6D#fOl`w&2!uA zRM|kli-~TJ%kmQAG8FCUVccSJ-s^V}(K`d3*g3B1O*Gm&Piu+YBF_;o5(W{X1@-mV z3-tWoum5p633VbyWv|}p@`z(Ix7)5A)hp4>diHA$=?aTJa5!@PjZCFt5pfx+bUUIg z)|NeJ7mqR&;c*$saIL5bTmsQx6Urd=`NQn--*)%y6eiDac@uz9((JBF!d=4-uiE=b81NsA$*w_npRi>4= zv*aE1)O=yI?KTStVUG{`;UjzIf^|KQeh;Y1SG|ggtEa}~l%~8lqi@0wL^7^Jz)=#4 z)JO(Gk0GPnR#sMm2)V?dzv2GFml*+pmd#P&x;_BQ5}%(FwgFxVQ?QLY5B`0AgV|_U ztyS5Uk|Y(`eD5O|TTRh{A0Zzvd2%cQ2L<8#+ha?z>TTTfSi0P>zDJ`&qXndoXajLWvD!Tn?t)NXt(rZ;q%lfFc2Hl{A*}YN6GG?1MPjj5!Rmq2-s~McIiN^Tzfj~5sgR0ph#*sDN8lkTmYbWSH#JdHkCXKA_~9gY0`tGhc*pP6qqq`&Ik zIbvI*OnLc%%ulS`!6VMVc+ULjVq-76{P7=wmeUDN&JWELE#tG}SO+e~E*KW&^cJ?5 zPFNWHdn#Z^^7dnQMP`3Jdo!>>y>L@&eH*!|VFXjfz6zRP`j?Or=8R(YbXgRn?1KTx zTm?M4(&`oFv1u`A-&x85va7BuPd zS)wfVx$Z{Pabn!({tTJJe{Z{2VXJA{r$2eq=)i5-BVtbj<|d*ghGJyj-?KE?cpirB z?$n-qsbrNUQN}M`RZQC*j*UtDeNp)8Czup*_YKtsXp)MHyQTDmJHH@dhb^GV`=c_o_ zu7nHx8cEVSaEMWzx~3L|Wb7ZeTE5=X+(wJ++l$}XkD*O#b_O;Er{z3 z61uNFuaUpl)8^@Zl|8SQ(HrJ69b>*Bn94wgAeLXmN~O@~UeRh4@7z?}5MDZy;WdBB zl9vh%`;XrK81?;P$E%{#nEz)daa&694YyA6xMigNT%{my&sl$dHMP^hs#ke@kYyI zyOKjXzh}4E4>hk|F4|}2|9g;iiu?ZNicI00N4d{9*=`=N4q=bDJlr_EafhYq!hN~N;m^@VsB2pC^FF!CN_l&!i#$EsQtXJ+U>2`70%)V?rbI5wlyxVIN z$<|FA^~U~IGL27GjE^r3HMa9|U5xeAyPPuUn8$ogu4B0k|Iyv|511AnU#u(2;YhBI zm-?ydHyybqLMj+4y-Ot%9j_=abV*+2pD)U<-nZ#_l;-5#cy!W?FkY9Q7vE6~(h@SZy z1EpNQTZA;5wy#>bGC~tmFB2V*@6bC`Sb4m($#8%kEp_``6A-zTBdPZJ1a)ff?$CPp%ss zzgFO(PVSD4oIs;h_ap2Dw-u=H-f&JkV<7a$B=Bwf^Go|Qg~+!h(~u9ee4hQmYj1Ba z;It#x&{vR7j_>c8!YSMT{6wCER&zfQ&o)t(Y{N=H8JV+<(IGggzhB7*!x~~9zs?-q z{qgrHyy#E6hu9&3z}%NFk71{U2%0M___N_wMzA@^NUd2;YwLF;Z_d!vnhlMNu;2?V zs{XZub>|C+6rR^fy`YySIGQ9z*}2mPi+OxKenNnJxkb6tskdh}%hYLDLdAOQ?H%S> zcyIn}!1nt}{VPQ|UzKTnmDqP`HRX*1{Xx0`U$an#`3n3Abyl!>uWWXw>Y72>f+(^zbjK0Uci{q?qa^Rlz`1?q$gMpm-p`|^DM>0}fa;pAI<`^e$IyGh(N z{gOd8e~Rbs^p~l|or48!K+Fz{h@pRc-jMImQOz4EC?I z)rH?49z63!vurO-{>FysRsLw3s$<|pr4x!v+s=daAew;D_YnwKP} z6*6w(o0g4Rbkdz+Uc!q176jxUP09RHfA}9kz&_UwT8Z}?@b#@B%S*nho#*C=CwHrB z^zSXEb;@NC)+Ec)*J~@&-BZ~H|4wQ;F`_JMt;!_%jzMEU_Er33xPFC#-3o&V>U z1P%3N1U6=IZdjOOC@9WKAFgH@((FjJNMig~;FmX^kU7oP`y%8zcN6u5@>SuDXejYP zzv3oAL3#<>_s>yQ3-YF%R zvtiT5=A?a=n}kH?r+yz4Io`OnyvH|wP&sPNJPLYw&xM8UzuBzK7eR)yQ?4;Cpje5s zUhwlRi#?FBYyOM?I?t{+`T{y~45M?n8GV?~|ZVkDz16 zV2dc;KwizyFD#}*^DC3*&yhWiA3hub&Scg2fOwB#{D_ax&Oa9)zaDARiJ`5R%Fkc0 zPO3FY@Paoq(5Pfr{5409aVw{H`8YY<_o5A$+=g)hLJS!P}{uwBk zSIF!Rj9n9QU-wB_?x$0f-iJ5!(jHiL?(w}Jb)9?FJcsOW8Y>Te<&Hkwu&P3+Ao{F8 zrCO+}!81=R0benBtl;E7efDetMz`;=x8F)J#iWR9+fJ}T_P`H?&|vD}*5kb3T^q1* zhgoE>3vmJ*8deXy@mnzQcfqR|iMQ%UDc(?~NW`S{?QY|n^NxQw4<_BAEovM2mb0_bkkwvOF()t^hJ2uz8+N>SEdM7p1K6J}qBOHc8!j;lH^;)vd(;F^491+zLkox1EBE=gM+z3}9Diw>#I94WoJMdQ(ijiM*e_CoOnM5tD=OnSZ7 zPZ8|z3*c&Yey&eL-TJJU+zpqEyic4yx}?uHtnpK_ku^%tdOsycF;+HNYEZ_YTO zQl)F4{p^X&DSZc%fTuCjp~=Q(S(DQAzdggXi7~U-AS*5=_y+TRo8D>@6yeBW$TUQU(rkVrx^>I+@-W{DG43G*CUnG^O|C!8x%G&f~CzNtcD>7B1}i>uczUTJvF z!cw%uCoy+r#4=%C;lS9_W3d7LDdQi%o|5Oez%a#}R5U)&umYNJyq+^t$I$8i!t~eI z(%CBjR}!$^jSuZiZk&*?s(t)vl>gxU$CyQ(Yg3=4QeCD*a}I3JP0g3*Els7TNy*OU zJlMGV=GAAhvp?1D(ibq$R91!??~#_?J>9NU`X*<6aw`4JN`<(};C7Rc;Mkjti|nc* zzxkdBFWz>1!{S3WyZ0njp3ipdTrK7#DkgUGf$-144bpiz!janVKHhwW^)loCx-an! zCnx9QrVVS#UtO0xVrLip?wzi!pUz#^GiNv@Bp5?NLT~_YKly?B4}Bn$;8V1z1rVBX zO#+v zygMfvz3kgJ-}pyGx0IBX=Tpv~a38*vztBL&d;TNWF&;|ePtJq87P=bBobTEn`6E@p zTrOg{j@%*^V_DATCpG)~AA9nxYuuZ>+q$3k?Af#Vj~*E!s2AR`V;wH1RyBQNd{Ljt zt;drbo7+#e6)p8YKjCH1!ZEfbV{%Hky>YgOZMxLmd^T!DnUo6xRk~WuXD4jjPPLAn z*I40<{>rLFM;ARRR8*Vx40j&9^XA+yUFiq$YD>QTx$c7LR3}rp2w`@a@+aSSjO@@Z z$(*f{{N^?FV&?bv#|PZ~wBjLmzfb_Wi&+k59JB5LyL2|Rgp@s)MIBYQ@6n7ss- zp(cvAzdudVfUrr)GD-?sGcT3Ry&d6&kNJ|{|L*$|w`g{J~?PQ@IHOB?xf z*G^VrI)oxw1$`=kVfg`)~zVRr7V`z!MJ;E%QI z1WiL_@7_J8rkqU=?N-RhXeIt$gmJEarf$?b4tW)FNQN@Eo`3$C$osFHk$S_&m;W=7 zq5jc-#1#J1+ZqjID13(yt%Qc*f%h*hh@BEGl>S!uN&EE$z|Hn zDK|(O8yf>8;M0y9A9pIZ5Y5leUvox=`@o%hd@6F578ab`++`MQT3TAeqob9MGK0Kh z*5a)Fr>`j7+-$$Y^M;zDbq;qch?~ctqzI}VQ!Hsy{jbuKOy`!T~+&aHx zHL+x!OUOCSoH>JS7cCGK9=r3Yz8_rpVZe&)MPH_^Q@?-D6g&-FKBCR zU3dTf{i(dpt}dOF67f6B4I-73cDcK|m-O6_eF$BYUCPQAD@ISAJc+ARWxwj~w!nP( zzUf&e17kx&R;XlR&(hf0eVt#wFRflhUA^i%dpoOhioH|cjpfUir{HGKedgljeb&~t zv%PfS)TvXh_rFg}ymE|HO{&9EEy8NL&a6=AeH+yCp}G0l%a<6ovSbB^d|G{skx8am zne1#II`H^3Ok(!?RZ@0Y%sY7BXeGS&YJB^AZ)AB4C@V8Y37h1(SXqX~$H!AlM+ZPK zR~|rn>tJ(H@~vC`69Wx_@bv)*JyTcc-8YIphQv@1sO~24$2Zh3rKMeky`5fbC9apg zp9!C%V-hGIb9|4^tE7p>y}fldxn^igG((I0#?6~m5D%)BIzqFE69J1^)_uctCI|-Q z(kxpRe#OMz{=p=)1;%81x%b_=LxbsDEf-VX-#-j!x=xl!hie55B*D_z()%^2mA8-fszn6y}DVzsC31YTiPhD(ED?jLadZfvf9g7$4?4bVqC52-?JBWP5wx zZ4t)Bc-DoXvvRHAew~-GzZ8>|jErI-x}Rfz`ZP1;f{_q2`dS!{969oSc@XVp35f&u zj0$u}174y-PbF@5Ddp+gxBDT8tYVaJ|HP^oJjuSPhqh1vRUY8x;_}D*lBIk1?(Mhh z#!M>n(eFZxrLYKnAEJP}Z!;7Y5!puq{-@2&hdx(D3LXlOoBjE=JFhx*cNs4Zc}+=! z2g;xF8Es?;!m%l6^02lV+t}QNM)chj{mew&JZR)}p(S_orB=a)D*2KXvs`(9urD=XjN(9j}E9dv(W2jF6RKSbT0k^6~eP$l%wCJ$|wlDY(5 zn?IZ6$!jfjMFFWu$N!a=a_2~m-V8>^9{FneqkZoFIli> znm&EQ*00*pC=gZU8x<90ESGEjRvEAp%R9F1_6cnxjt&l?#--@2R(^Ou$GI|*$cEK; zx}mjI(Zwatc->(IpVXSS5GSF}ORBdiLz~1cm&1n-Ph26rs@5UWJ!Jum{0k{tIvW~& zb3>(j2YLAfySlo@Lr?G6u_HN(}tGu&}nT?0fd?Nh)3K zyp(?D&YjA$F26uXSnSNP(|z;j_vyCdJ2r(?Ja5XiRX2<3U;1cPgkLwsG)Fjyn~Z$* z2TfPqqrbCrLtNMb25h}-m34f^(Cd_xTHfE+$2(EeRTen$xqlUFQE_q5`)LTq(C0;j zglsU(wMlN6eH3_*@*GdnM5plZmlR#Tq>sm@xW%DxwfpPWM?x#<%L^WOd~Ms2R)6zp zg~zE!oJ;Ak@7|oP!IG+1@W{UoqFccsyEWyG%#v*rGTb~oFA`p(s&pYKDXDJk7@YP> z78dI|^L-pWB_VnZvx`)%0BrM2k5)$TFLp+@k$QSe^UHu1BcCpbf|ycwThgs&O;O1) zOt<=wl}zD;WXDFRVr;s=(1ip(5tQNi-d6ALv3y;YwH4ovvq5Xr7VBx&&Qy(cD1+?8 zW<&u3iAhPx=VFS4!tFj}DX|WM?-~R{sF1D~$jGm!!fPetGWb9(E~_$@?2c=Xo;_dt zEo=&&QVE^)&#_)*h?RZSl2<8oTev|wOi|jvbu|91+|1-|R*l0CPsTa@giHr>|N5^` zr!Yv?GO#!)hXsbf$@TemfxmRcq4;=n5?wFPed(BQJ@1|%(EX~1SzD|(?+Oqliy6gZJq3y2j$L{K9iGY_GOwW)xp%)6}dWwEv4snuKuuB z*YMM-GXo77!a~MHn=h2BhO>(uy-eZcJ8+i^Y|(q{+&z8BVweDJFvFfR!M2cZPuJtW zvR+OoM%;SiVQn{uSoza42d-V;h&CTi@7U-(rwM%aIn->0C^Q&p6ms>d>67B5^R?XK z2-or-JKx?LFGTT&lY$Ng!ZZe`B))Mtc<`W4;MLxD8wOl4#Q}lGD){TgrB&GfGN-=? z*q;TXe0CwF^XpgPS6yM)t5C$+G_RX0V)Uuyh;~{%$*sFUJCC14ydil;;S=lwcMZ6#OxK{u0mw8j2bE@byx6`(n7}u@Okc_&<2CH51?AzM7SMvj}p> zQ{$iZS#P_DNF&psN3FBBmxuCEQjo%MAWcuu-UvK2;v^{yn0uD!9~{iA&|U4;n5aZI zPagUZ*;mi1??sXwbq1|r&oKw23^6lpY;8w>zd^@|lzN+fv1UArmCVe%7Z^55OB?3c zHfT{85C{u9tm8WibW6zrZw{T*QrL~{yNDuU*Cy-ZjL{gYD9_Qd7HD)w7GTvi;%Y@iL|l2q!{3PQhQru8%&1JH#V)%1(TrVPb;kQ0&YsY`g{@Y4aM|Q{#8iY=Sb?_j&0XTzmx`2WvFDW>^{`9fY(Wmec6iT7X5MO_7Wl(tdnp=17^clRM z+N@w~H+E?!8p_tA)oXnw&|X1HF~w8D7Uwqf)7YP?tAdDHIZ4Fe;xAqIHMEOx3g=T8 z6jZ9j#KcZ@m#?He#%`zo+<&DV;YxA|f2wp(NUEn>=$Elu^){N3N8}iVTW9I#&71co z3-`+|emU|8B##(0IMq_qT zW3DZ;UWPFl_%|Otl#&MbZ(Wg^zNALiI-8Yln5*Ekn11{{9B}`ys_!=aOl#Nr!vW4O zpZ0ok`SLQjr|u7W`uc0b)wo&3t-KKzeNGRnuy|9jx?SvjM0E6}?(S~c>1p^=6)i1g zWKWpcW$|DJ>LiO3+M|epFwC-8O6h<%f0Z4pld1?~bK`3?0kyCrNIC{Nwq@_fX4#sP z*miS>m@QmG>)ZAShdx2``ZjC~g2h~F*f3zzRwU(JG%HN;2I0Ws=+RyS<%U^E68OSV zwLOA)avEwnkil-S2MdAMs6f&{(zyjop4QhNu!}?~n0xo>yPH`^4|Tb44(AU~ z2EPq+s&eircR-pofBt+N_oWC=bkp?c(O~%k`Qx(67!t2{of>MA!1mLP%gteAWOSvt9m%{3bGRln6xgvEfHCM1yA!F;?XWPrqz;$j`Y;65rp5PXH*MTl=`lYi zeRFD!l$T`qPQMlF7gHdk`{C0Mcv9W_Ml5Y@ZMfDhZEXg0BJD`J6g=qF9C_o$mAePh zo>o=S;TEwX&q2Z*fqXW!F@J^?5ThP?^(uv;EX+=k$1|f)Rv@U)yjDrKVd2U468aS@ zR!|C1AC&2~DpTn1s zg!0{99D9tn8ryoROEFg3nfAtw8_EMMg{M~_d42Hkp&t@GxWC%$V|(4f#CPi%mFaaT zVE6R&q|mnQeepTm83HHsFz3n-LC-FsnrZPzzb(p*Ly;bQ5wtiG3n=KGWmXnem@luP zv8<)FbwgBD$hB+!5U;-JI@a4*;CRPas^r*llP@J2T3RoFb)Q4FV~$U{_0Tp&!F7fT zXON^-JlZ?HdOGh+_i3Yf&&&_$IIEZ>t3_-*_L^N$S$QL{P26z4Lr;J> z_ZPJB1tIBG>G`m?%=|-^yj(>?c4XJZ_CDw z3O~+HpuqwSPYZNhu@CYK3ZB%|>{T;{!G!F$+yl!hrecSmvIbH2NM9B&ON?_FyO8O%WUO*vn6p|J>eBu7OIBW~R4UZw9V7v8jiO5gcmHi(J zEI-_?w-Iq0w_=xaScvHnl52yZ8uBXU2#^oNRD0CaXp!<~+O_Y@v8kuUvccc(A$nss zzOpper|jEw&$yTs9zLr?vMF%fN`5qmbn$=EsyH?8{JISt(k$++dG_!h9U@ zg~+SWr1I1%8uBrnSS+RE#|1Dl_NFf1DMVa2pevwG`bB3%+&w&i5YTH} z+qx@5cDmiA7!4@f?K`~RMS(v*35Q0Y(dW;fudSA@_-kh%!KG--Ce~V3?wm4W z0E}HiQArT{n}k(OzI8F)YsP@D#xtu3hhruAf%sqMu^J8^^T#z|C(1JbDM4rTny4i$ zfr~tDz~=fPex_E}Ctdl3AZxrB+i@5M4L`IKQa3(o$xk8H_rPt!CY4=o?|ULBzd=Z7 z`K3#j0Fga|P~&-oGicdlDxIAvH2wf4yBV-n8XB6bJNMGFia!QKGYqkx`4o~nrOHmH zTH)kiX6k`=_YXIlZe!U$>ALaFOUJ4#b)dA^ar)Uu>6L?6eenzGtO_8yu?EYbhy;6H z_cCR4n6Lm+Q$)p$7+oaQ2l#5fUml8)a^yjOA2;!}@~_K!aPUca2yy35;Cd+LFHUve zHLW8Q^$ZVgP}*W0YQp@eS`K6LvU&Vuq(F#p9a7xTqeqaU?S|z40|ZOrc5Up~kVrox zD*GB{)sl5$qJ2si9N(=PBr|5WyPsk8Y9BP$aq{ue^6TA`M`#%oc?h*jWw>h8uQi6JUmRUE$uS6dEHsW4H%)rdiLAkIU}K$h;1g0T6grtSC@f3 zv#NECFtk9oJu-f1^nEWsV@5&(XU_2tt0?x6M6xcB`t`g-kmJ!uo6k}&6lkpDzBESa zuH(xGl0(@#BV{gU@B(Th#8`{N{^_qA-sjgls@HF^L$E@Qa(|-s7thMz zlaUvny5G>{W5kxBtSO4WT)!oam_jgFKLWjfhT<;+vLBDsdK-$onb}6X?bfdF-}uOx zlLp&>19wgJ#lIwb1bK58R_XQk5ZS#BE-uQYp1&w-JeK*~tHM!0*dj~0vL}LBfkwa} zYoYv1@b#_AH?>tN@_UPp0hsR?95nhg6xtK(w=5@?2wqVh)O4w-Kk}pE}q-lG-870t(%6%hmEmJ=L zA+4(J=lynKB;`OF6BZyOJp4jclrY;p5LabkAV5Yr$0}hHZnchYyLr6Ww=jf4TP-Ws z5Dj#zx3}q0^j-nvJ;`$5UFrf}US3)ENd`c$V)|aa)hwGM)Xtnen*=+>ZFuJ%zlL1T z%f`M-zjTXYB<+?{{I6VzjI2*T#0)2!?=`z~Nt?u>JR_e}mjIX(##G9=@-FP^0tyIIG+tg_#Tj@;3(BMGE85#l)D*U) zn!ol;d~?1p(sHkk&MJ%@FhGE~04>ryz!;1^SuBn#da;#EV9!~OC~QkciR`H^uqV=Y z;m?z3_xq`}A#Aq)^}~7LqD40mgC5NdX*kORq8yfQv2){W-W}p(XD^(PasIppwUS^Q z;ts%`j{z=LpH%;T=3Jt2JLka1T)r$+PP9W1R4t)AFn_t+XIFoDxyny$5^8xp^2~9| zh|DmlIFwVFWvqSIg zk7Ue?AN?VV;)ivfcS}4kd|mfM>HMo>57$u;4DtwqJfaBiNa^K9c1d@VaZCFfI1 zffyoJpkn6${eDhI)gpvLrVKe20~=c)Y)uEe5(2@+q(82&K+V|eD-P8I>$g0VA6|t` zc_--X?CjugOo0fxIm53em!5X735QVpD8N(whbPxjew7?YRz!UIg$oN|Ba}0=fN_Q| zyH+UtDs?ya-qReCD8hx6E*N#IK_+L4)P*wME;jMGemBSwu%_c2>NMT4ax;*vqu|LZ zHZT)|h7@i5{ZMNmGm@k?)1|pw1KJfx!`7oI@G^434e`e3y?szK#Pu_;*WSX5c|D&=UWd{dp_06xQ(Yc#gg>nR8J-2DWq8(7 z;PJL;SN_)2@u)6;aLH=Oa5ImdsqlBaye+N@+4 z3aL301QK;1enc)j6dKqn@}r7R01?~!W0^aY8K{>75Zi@l2GyImxVU@(s38O4l0ase zDbMlPm%Wde1DQskd`d$}G%E`*!`>wL8lXr0oDK5O-FQy$yCzTzDnVQ zxRllS?Bm8peT0F@(gcsig_-eC7*V2UM3meGtJ8($Vg~xhNFg!C#Alif&q0mgqoSN6 zNCEs)QZ|pk8~LN9mxi(twH3e$k>eXSH{{v13qc8J6Z+ z0psDR0?4b4aKs3$wJSCX2xuTB2tWxcfLlG%ACNxESSUA|wj8erg1@G zPYWif@Pn_*Gc);d#VL3@vL>X+1+8c80WKdQ{?)X5T@5ez5sx*~^!GAJ$V8VStu>esyCLs z?*a4Tu~d(Z0TkoXOEsjcv|aLp)^3Nu(i<%g1vy&7RJe`3>S>Bo(7Wn5UiLG26iE#l?n;`>7hvx`s;-JALv4{B5Z;j44frOSX{6M--&4JVRE^F6RnDW|H)fjXQ-x(<0NK7w@)U|BSlV=-$)chs5uB!VSGN zMwk@|!_TvnlS^iz|d!Qrc$a_LZ{+60rdFk0gxl8%os z*NPBo(aa;|%vFxlM9aV}p0tbTrW-P17rX@cx-a{vX4r|$`0CXxESJD(?Lhq%$R88J zhJC3z+$8%Uxj25b#{1R+M_yvtj2<~H1+KGq$HW@U1kywY*^|%Cl`+5{KTm`KsRd$D zivkI<@A*)wySyt7-t3-XF72Kx?7Q%zkSicyf)bImDQ+L|nRr3*IvE#J05-7nqQJUu zxP9J2hKbaC_d(Da2usxg3=w(Z0)&lrIT~}g;bH{9WHs?H?E7A?1F<9#XN9QWjdNt| z&a=hj#YtGoF~Ns6^?U^6b@MXaBBdM*43fmofmk~T1#qpL$&U9e7z*3w6&_xJK2;kY$@6_9jU?Z5r z7x@PSoKH*R2PKPz6y8$~7v`1WXjnDeUx$Zwjxe9v`&-F|Co6+kL;(;IB>}0(r>0zf zk{aXG)LHG=un+`~ONjG3z(hs4OaW=>R+ua`pQ4!2~!&=DG`s-OpdY zMj*A@iE`0-(3qc`4!;}&0xec7?pQR6rK-WSI;pw<@OX(QFe-3ZgexKF+6|z11mHjy zA{THqB%8GGWhFB|YzMKWVHM#H*CQ|r1U9J_?3kmtj!))aq;M*EYun2+Hp}KnVSoM^ z!ubC$uj*d}`~Pc}*8lCdrMx5bWIzc1*RHL#^;6ipR?2~k{AYEw2L5x7(iVOPNiaDT zq@<9$`z9~J0&&)!gFbX`9qr1MA82H3m!LgF_r2w@V@a^~6mOX0XHT9ysa(kzjBm!c z@+@s1OEUHl;Stl1`;w%zFy!_JvWb7Ce?naOq8j=r&quza@(K5^Ed z+}(4Xby)^TkKuOt3CJHCdzqGzF9DPn2wgJc{2d<}KK=rgGT==~3iThbKVswJ=yboq zT9L1Z%A=Xq8Ezr$WQ3V8u#W*T-&?1Z`N(C66*lAoFndFpHQ*<%r1nkl9R+_009vT0 z5IrG2f^XP~IR#IlVtg<`nuUesjVF^Xl(ksuA~;4yM#v=w2i0avjo{&4qRf?v5QY>m znmXsmnGt+Ixd??H(6P;fFW^hOgtC?hKo$Jo5*#R|54}KB;Yxx5vK!ti4!!&lDa8}q zEM0C~R;+1;PO3dt40T6aKOGBbgsk?Q{dJl|y2SA3KHIP{Q72(M`2K2iQEnc{^Eut|^l4$8SX~EX87nzBt_%(y|LbAv-bLShA`Tic zqG0Wam_4VFg#g&_$jB~0p9J|v0Bt_=a|(#U3)JH{!zu!bTCq7g5g9Qx`h*6XgMGQU zFCLKe6U6&+4Q%eGvl9D!g`+rqQx3Jjteg&1qh7vcGb8c`+@Tc0kq+@UI22s zg8Msyz*)bAPSTKS`;BS_oM-w`4B2X&`LfFlCHytFC zBLE$&kW*uJ2YoNCAWM zs9Yh7*vQ4T5H{8AlI+`Y?5f5rOFBYc22TW6Is#=MtM-!OT?Am=$iuT3jEU!9d_AzJ zSVin9NSXx0z(O5+5Kp-p8hTzSkf{@!@f;|+#CCiBJcENvif?QANi<70OjXa6S+&7CLNmwL0M~imL7yZ-7D)Psz_w^&+VI zy>UH(iU67ge2@rqp_N1={@u5aK1J zS{!Q?155B19tWaDOoVPhDnko938<~WR z{M91(=1~Yt4@Zv#a4iZp-i3uSq_PG5o5v_{WCS5nAOjl|5x=iB^o0Vq)o`4<-KJ&dPoCL<{LEM6z6!PNFk-*#%Ehq)hn%1EA zMS8*FWm^sV0JpwP%XS?*I!_*B&Fie(0spO`Jjbc?Wl#W>?wM5~y zVogNya&un-V#3;brMS3Q|B+Mbu0sH+z!_0gypSUxCbl0F?Gz~txC8^L!fzfS+oU0! zwPutZSAz24HspmEXukI(L}CMra1~1;vf|5qs6uS7SU;Y{kb} zv?gIkryab5n#Fu?{Y*1o(xsdNslXb47_?$($&)P%U@k1S+#xIb9Q8)^E`=46(cD?>>l8u0FmuKRhO_x*o(KE0p*zwN&7?cVYm z&U2k>9qTyuW8e4H4y<6^`t_3)_qIrBQw#(X0PY|XcLu39>f{*pom;ocA$4txNIJ1& zJt^lAdr;*c;r%)^bU8g7b+oB1nJn2BH;Hzg4`d?&eL-fN&ghP*x`+b%*B`yG0Bgyjn1XVtdR<;$!$jmVx*W!daUli)yXy^Gi z9k4H#QTE;<#SY+>kiE&TpjhnzD0Vj2ayt=rp&Z@1ZQG&!JKlUOcHM$KupCQ-a0d{w z&*eGxBeuOp)d7;#O}Hj^!{+gNZ0t_5PZKsiD7u>j_301_jO#{`sCSQy+(mUApJ zE@3%huf?}sd9o)Vmmo4DFfN7|hZb@FCYmTnQl5j;^9Pj?xe560ig@H< zqm##j+6yWHiN{Du2u*JcIF2s5!Pnp?7n2UtTN&ixB$Wl%AsRG`Teok&LeldM-ciF@ ztf&@{RqKK{{Q}tsvSaSTEj|Z6WbLu|n=IJiEFS@CgHaG=XJ`5;&R?UQ)JD^ zZpG+s7Q0H3W)`}Xt2c{rBm*{X;Cl={4C-)JEX#5@UXyMDyytbXSA+V!j09?spA%d9 zxbGmMAJbz(FqcK3h*07tEvj8QMVrAn7BX&5F}qsy_%XV5fF69&WRJuydA8W)faC!n z;;0Mcf%B1WI*Qw9oJvk~AAtH1!v#QzUMEm6wulrbh{MBK`M_F`vyj2QhJ#)pr+bd} zADV595aNkqVG};?UAC2BSlzui)5{aeNoq-+Wf4pLxNKZEYnOW6w5fa&hU8Y`ZokC2#uCt0tKw3t|dMwrEb19FRFCbde+Q4^k z8I3f&+|Fv+j&;k>NP}2*`^5{f!f~XrQJ9<#Lc)Ut5TqEu*gV`PU`m9@cn!S5Bw9)l zbaX?U6CiB_P*hh(D_2Mex%-Ze+Mcntt7LubV}X1RNTHowaLx6*e7^32B64!Nx)Bry z=&Y|zipghqRSBh7uAO9#MEa@RKrz=YjMbU60ot9UZ!t6gWL0c8$VjOjH9% z<&Ia{2ocmo2YJe1V=h>*6bA&KPYtm35JncYXMc(a5?dPSD>rHJ^P*;tC=S5MCc?9? z-(XWP*kWje+$B97d8jquAzgcv)>i=wf~8Wc8&+H_1z?ELV}Q&vv`_9~l|h;z%OA{i zyh@&N<3?J}B?R$7t}#N`Y6M$(?lR*5rrukl8O*#soLqN3I-2x}JWzZLCf~bzw*>1O zdJ}gPsEa61hI%{BC7?LcP#_#0!T?VW-vbZ=a<@YzuX>6%VIzPy3S&$f5XY)IUG-*X ze_x?qqIjr7o1*ZnjZ}MUYY(<&>^Zcxd7**^cVr%;jj@O-oT@_uhlU~>(kpLP^QmO2Z90Xn!zIZ3!cfX!vtJYx3VMtg(%2L{F`_#Lh5+*5gICeKrsPXOCbrsi21g4* z2DKH69VA$^JJaB0>W0h>TMYdJW~v*A8|8+U)o1W^YR?iRGAQK3PWhO-;(~(f0IAyt zRYPy|@In3AsyDrjY`+1ZZ!&9d7O)jtB*a`*7?NkNx{ z9Wb3I*naD7mu=3o`UtK-Nlnd|rPq1>BS(+YVb~3=rEtQY1ekfN*!DLV2`(O+cG9x* zaO24qE%JN#(CFLV`)K%pw9mcaj#D`k`CeZwMwS0DDwk^BY2m)#4`9WCi9)e=RY#iW zBsQ`n`v(Q74Srz?TPY zoMY}rN>=prkXn}mnO#+5!p6pi@xP9LFR84o?6-XqIz*+7PRk<08bf~HuaONJD{bX2 zks?41yQ)`RSnVsou=f6fP);EdP5Z5oDs(?zhG=ud)>d@U;>B3Ji>P_(jSzjGKgz%d^?eU>=i_7?FmJqz(YCP<-u7 z#(pCj8X8KdAwW8#P+FFyZwYF6VWCHfJGTQSzUX-V)@aLDbV@Pr_ME*kJnGZya_Vmm zA&>N11`{3$;0;nhW4?%L@Giv}lLq3Z#bMLQf?-0c{nF&qVhdysz zX-&cj*)y)R0=PvUewmU+Q{ecv)mG?lK)1$@E*Ij&o`=f?_wQ#xbxy166m%4P0M8pY zBJuqXK_i9yv< zE26b0Wj;wXfOPsbiAzZ6L30W%J?;lkx=@@e#sTJ*)$Wd}!iWLH1_EaVZ6N~gZt+J5 zl<3Wtj}KJd%?ty)^!ok#wMgT!WjaOL8CD|}Kerw_9X)CNb-7;c5-9)4s}JkBXl~XK zagJchh~Gi%%Ok81&j{)w25tnTtz)RQ5u-sT!ZW!4=+PYrQ;%1oX#z@xH!Abi!R(a| z?E7qnRYJuqWF2P4dgOq(twR^ds4*c>c+%CP{K9JBD})PyWsIV=gj5_hsJjT_0rt{U z3p%}mf&!8U;;ieHI9{C<4)ziIHjX2^T|^;y_XGyWa{ksmWU{zWZ{O6jatKX?-9P{% zq=90NKax5fIF}GK4Sb9jN897UashaI2iV$0q#|Hw2Q?99ANp)i{SjP7H?RRv{CSE)-o`-VZ}ruU&kI4B;RXiZC#2~2}Dqmff7aSitW z-%WRNbJLN7oKY3*MGFiMT=7xO7;@pm7&yYjzT6#!`ij&9;0x8I8yrW2x{m9&e>gCG z0%7$xqx$a|9rXng@e-4OFuFiZM|k~)OkgX%9&)-(lGg|gM(ycYqMxDH#2K&a0chZX zF(TO`Y!U*8QmZb#AtG(J(9*l8PzVS+2vF_*#RW{1cMZX`kvYgkD(PZwpbdiJnGQ`? zg7icps$$-H0K_0gM#s-5?Z`x-QQ(S1{$PZ2eUTLHx$t_)kkglAc>~b1$fKA2{<+=} zk0YY{#l>G?+XHJZ)B1WIzKsweh!h*W!+@ik<@bV33WSpeXo5T>4r|w~b4NPQB__5B z<`-U%AG0Ez^O9@_?fDfhd<-V=7ZB$N6B7ImiV`TCSVzi2?2crrVx&sp(tiVv6A`Cl z6mTDA(Epc_XYyhP=9cXKT~tCyh|!qEde37Uys6{7gE0?5tC`fFvat8kVs@0HgH1zW2{rOXD{+%PDdC@!zHep$z#LNXVZNzy zS!@8`f`Jr&Kz1*H#8dwq4Q~QYlB656?Vom_O$PuTECnl$kkJucUZYl{M+b`p1-rYc_WGacVNf*y!K7Nuj9@l z_Zqst482>wCwoI+3V_%LIrlOj77@WXJCv>;3+&fpdY=-aaYV5lD`uZkM)+k;Eb`7@u zWDvCaeP~ugPJal}1tmPq=vtNoz+a(_sVj1J^jP9Sq)X5fArxXQhsbnbizh8%M0fc5 z6Fi4(ItVtpm{vsOEJJEv&@i|1@vTJC5*ZmO@B44DLgXp1o&YhMB%A0&A}eQ#GeA`I z5C=_9Pj6~C1*HKtZqg7(ZQ7;lj4n1xJmnm7t?A#tKjnW#CmQ=sb)sF1{~L@EQUYLFRbX+Ioqfkl6sCV*^|}dVAL! z+K7cf5NCQ#F!K7_%_Mq~0=SU%IN9v6%aGpB*=$p0@~?b{VgDGzHs5biSi3{mZ+qxL zXBOibSz6 zMidL+0RCOV=Oi{`K9U!2S<3lbcYthgDd60;t#rN(tl@crjx&jE36|aEKul1yx@jnq z!a~V4LaQ$W^o>+RDEaCL&fCsYS^hQv3}{Xu7uX`=)RG^)vlKCIcuAUds3f42$S&=@ zd;7KwP~h~8j2__JAc~@rh&B*_tOm$Pp#k_Mh_GX0W743LTrgBm+pM8y#@vUk8L2j7 zoZ`auo+C_jOjY7N0*ncg3%866tLInt2?-&Rtgj)H$ zu`4Dhq_5*p2*&HzuLA=X%@0d=@;-rs7FQ8;v2U*=cdZHxDsc% z9qoaY1K5yP12^&{0dTiw<{k~v;%v*xs;bOZeaImzh*>CJ6?@S9MPTerOf7OZzsL`u z1lO||j})y5(%8q@r~3$ck3FIU37J`)Z+?D$XVP-Sz-!ZEoY4K=nzUn0%a^oUr_n#j}UTG|ld-HWElCbS_Hq zWYoBXL>U-!KsZ~h_r_ZW?O%$Bl}P)=%WE0f=vPOYb`$;%cw@5Yj6c6r%>wY6Ir9TD zh|5i|@wp8#F7}dM)Yj0mtsu0r0&IZO5V)i3^Vsl2ZoA;k3?YWKf!Ygpn7k!`R=fa8 zET&}T+JQjU7z}r_i ztaLoItzP9E2?N-R4<7q*_IoJ+8p#&GOV4-Q50xO?}m%0FaB%`{p&Pht9)yB?MBp8X$Q<6r47DFW?A zT~~shW|~PeKRB*r;DKMny?=VEppCdWE_hX!9h%xK1+N4m_K~icu!ThQ2;{S*Zo)>) znBYPZq^(;?1Ow@tP{s6}JH9hIJ3GUwWpT6tc?959!=B~XJ$E=wkRL^9OVb~E^q~E> zTwM9~gEt0{=8)BoHV-5luiD$&^;^M$fzxUfl6mAy%P1o5MDlsl!(%aW81?S_i)fgm zKe8NqIr$766RE2p^+F^Nfoz`WNx`z7&$rM($LC*IlypN#-ppUyy0XgW!$XvdN8hbV zE&9caf>GbPI|)P}qV^)X`2v`)+-SRr#FcF7sOm2ut0BB0!n&)gQ!2n=$Ct|I&8%TQ z3!_Vt=*jU16ZPj{dk&;cgd_p95VaO9VJ85+U4zN-21UYT1(>@e(}zy=2+CO>^HcFE z3itjo`S8rgy5|SpfwksW!6xN${_FGmaB;de93M;dS7j2D9GoVUZ3Y89*ev)Ub>E5v z2<=|`CnLcsq1(432}idJ>F8m=DkNd{H%R^Od&m;%Y3mY@9)TiEKF}C{0Q3?(22Kb| z(Lw^6Cy4I8;5r~Wg!h2X9z`MdBVNI?IxylPp~^ta zc!^o%<6BGkoZ-$tm?3`mM6Z<5nD})KHYKxy5pxsi;5t%aLZsl0W-KAf&kthELqhV% zU=zD4O)*e}5jN4}3>XD2D|Hdq5jrBT;oSHeWJV6TQ#2YA$x1N&yCv25!3F zaK(TnQC9$~G{~GSy}ce?C#10tz2^|sz#%|4X`VwMa9B?VY@O%wzRE7}f*|Io@_0~_ z*{Mq0L^V@dJU45SsXgC(0@Z{UJwQGHPzmZ?@7^6l>rWM0&&MBaQOFH;zoakqC*8{2-QD>nvH}BY6SND`1m?J8i$d6rY#&4 z9V%CUGpK@(wO1Wj9U-w#1jqynZy-8dNtu5DCG?_E!URabVSrV{cpI9%y23Xf+{xgu zo&?80E6zcd?m0la_t|FW3cov3qzQHr8`9wsZx#^+iU9$KgpPSTFem^{Ye8B9Sbpn8 zJv~M9eek&>7yzxF94d}=M7sbWnSnZl{%*{E!19%Lp%=g=*ytl9K)#3~t~?MK>{1HD z{2d8w^s%F1K;z?QlPeB$duoveTx2kr&lEvXW&b>tAnIZe_(LcXy0K;m5t?P6xi>IT zPAqN$_Ts-|*pC(LeoFCxnunSWh%GG=ZY6Y9NT&xCC+S_2HWSio4sPzZT;DKR&=TSj z72Sy12+4c2u%pMxV^uXo;0_llU0gV{@@v*WqOt}B6akDe!M}kQq6^v%xU*sJMG2z; zdp`{YdItqRIkFzm-5{tnkt8s$vdo8u8(V=Fk~oTBkjl_NlVTkTl~VNRbfJz?5&**v z*9`b)1L|6$2=vep^RBMZH4n8H$MJ*KIhDu+Py;GOIG||cYp}Jnvau<{1Sg0l83gp` z2}Ilag+y==DWeF1fkHgOhQWNuU>?C!rS;Mq+fv$pGunAW@id5lJCMVfK?e2R%SS z_aj(5EmUTH)!JB6%Xd5FnH=#-0iny*5GY)AQ`U z`a*HFh(am676b>P%ERR$Rr42?Ff>OFN=Hv$MiNDsL19~rz(jR0_&Zkbs;UV8nxY`E zki}T9R1pvpd-1Y;4>kQ5S(<+k{OnNkRps;|HIFFDMc*3^kq!NMulLy9ya|ly zSOym{mb((<4yfUM^0|F9(lFyt1R^lh*jVl4Ha9v>bd*g%H;&tFK%#8{ouY>swl>gi zhy@kcg8o4!!Ym}AE6f+OtgVaqfSnXk1tHOXR8P6DsuH*+N0u|{PT9ulF>O(z7=YCaTnz&QaOnIvM6{LZ@^gHEw!JAV8q@Dvhh5ilpTk>|C9aJ{fGHQH zA_eBc{=B_Uz~=sxdruy*0Sh{C7^Js6B^xWg&q*sluz>Rc6jjIqceG;>1!M2|_!UP6 z@H(&B5_JLz)5d*pttbHN9By9q>A)j((JZFikc$myxRSmxgo$us$xK3#RIkMeO3&#a zR}BXAZ-^Y78bg9NZ#(pOZ6su=M2$%F^C%}!Zd4das3xeqLY{j2bIfKsg0w zzbxH##MQw@aloq}J*}dM*8M#^oi2xg?*n{WJ3`{>ijq*V@9P;Emr}$w1nAW+mMhvw zTXxPkVd_R3Ahy@VpV{fJLlQq&s_w$JD_yh7ap?d&pTfQm)ifu?V{(&p1W?^m1JL^k z2_CQ*(V0VYTLQw_K(1Sc7*gsw#d!43^r(6hlm(75>okrVQ|M@P-mK9-cov0QT)Z<)imQkLwA z7mOXM0roN(Nv){BsD8%)P~vbCdskhps8H~}Y-}3{P6+0Y{w(a6>8j|wLtR8sh5Byp z_zha!U+3euk)!MmZ@E4A_3J(26~TS3Z}LJHXDKfBzO2QEfg6StwV{w1cgrjE#gB%_z_`I-a^16hN`QkpQ1r> z^ca)8YG~go>38=;1`kR+d&5lWA~K17YbC!m?}V?sSID;9?59+2PNy4JWkt?N7R1If0tg?N9CZY{16s_x6wRF%NwO zkv4KFUM05UJ{bWq7PMi=wxiL#+n(Nm-Lx0=JtbBneMm)nUS0C`O$Q1`aJc$2?em7{ z;4Azr=e%8d%HyEgP3OWI36o#CMrh0NUm;<6eb-T7H=M3Vjz@qT51=S3cHZou@Bi8&o|VqGp+O}ESR zeSx^6p@acr1rK-WrK*RKQ?qZ6E5BhuHMhNb2MMtmH~=B4KDmF^m`D^f;M@Uf=_Tt9 zPy*J1F-4&fmE5_%BWWp8z8&oc(F}TLlegZ~=kS@hYrA_;8ZV_jcNObA+L-=y@X1#A z{VV&ogK%s2r1nwfKhHlbr=xobsyht@YDVxArbodLu)vG;?Ps%DB+&JaHo$CY&sgES z7Q6zNB2~Ew=O>DARM4cT%J_p|vIlz-C5=6t{*qB$SaB@;h7TU>G-6#U?z(J+3ws_i zDE|@BE1j!ePKe1JdvlowBMG7?vHU`-Sv9J?$w!E|$so%-TnXBCFZgpi+S6`W&~kj5 zSxrbh^^W_c2mlS0m2T4y(qtgEXcPljF)lN|LK;`&+RPVP;!BBC+vDztEFCm#%fT&x zE|b(toxiZd(@>-iBZQ62TzKu6b`^E>i#j_wQT>v!ds6cS?vz;79`FLp1_GRJ>8-V> zqW5WtqE^HbOmE57ar!2Y1|bLj0W?OzYtQs?3!Q*~BoV3dQ3~BHGw?-7rN!>&t`I}8 zm+za;<^f?L{V-xJ>>vRi!8}UPN_Z$EnKZ4&1Pa*HwB3-UYxH$_H^87fhP8FPCJMLz)Z-5!4o2P^VE!BazJ>wCyLOv+|kv43*0(dbc{(dK~!0& zZ&}ewC_@{a7;{2W%Lzwr0LO48+Xia-ZJ1kKfF$;YZ}}$p!_D`5k$FD1{W9;=g7`zu z&Hz)@m-K0o61t)HfpFn{GPMPmGeJ7YLnW^t^E$NstZCCBphJ0VZA`(?6(w0 zjyBeC<`i?o{mKDQD^`-OBYHSnu{MY}2rwnGCY+0c0>^w3i6Jv{yj_a%3CwBI$Ro`w zF({2l0}Y?lg&qZUjy(FcM52boxCeBgY0M77svuB!-M5G1K*osyDM2g&|KH>}^DF|> zhHyE_aS)((b95n9n+D@yFXk#*(aS+^pxjU)5s!k6cSQtH$`j=aiRPd->Oz`8AWs~e zB2C!hD7J{j3!x?@Y2|N(89Ja25Ha?)X~Rkp(->hQi1gn`w+u0qI9XkOjLLI2I6q2U zztO)FJxUUwR>Cc(=%`-Q;0`13?1sZ9k&NokAZj0x5y2Y4q?qXScUGXGX#nOgbNYo=9K~k9Mmm9PApndAUHwsyefNVaAn!fa3?MtXTwR|0t@$5 z1S)(x%G;=4ctHg#<22$qdxz_VY6((rv^d$H@>qDeR=JcbS|3FB1s zPR!aY%euIzJxoT|*V)A584do#56mVbzRHfvKS#Y?V z9*K91Ab2o=L7?oZq0WFaH`anY+_@WjcinpJEriFQmS8tG>rA==l9iSEg2Z5;@C#@F z3m}aU^9T2(E5|o4L%AJB9#vj74oyF*w+H(}l#CV%F#L#5QwMBC<@ZpG0N-T({B3JSfTx^wDK zD>VyLa_Dt402Xr=|o?ikw&rL$8 zMIWAY)A8tt_mv*N7)*T;d-Yk|NnMtS`X3YlL>?6wb*rW2x}8*BpJeZjn=mM>pUB+&0CQNNTyYD_`^ zuwlZfi31&$*mjZmkBPICinfzci9fxRC@EZKCe=yj0ysX5QK^W15+2*}Tu>Ym=p0e- zC3Y$z$R_-WnA5MCo0-tACMGW4KTH68(LXCu;og7<;sI6y;vEUS7(A8JBTMTDjgL|8_!B4t7HAU)qV&z%rN^ue!RnU^14t z(JVgUs|=9m*@jD@kzi6eNRe6$lcF5y7KwiFFNl`Cv4ZdzaBBp0_4q!4t%_4vcmtX> zJ95qdf0iu2PL5oH^#cp?sdAv9P;L-P9g@Ik%=Aq#N~QYxRzbgyX!{2A)T7Qp3VFnmYOY}nFF26u>IFV*cj;s{* z;4vYa(J6w$J2>RMw?VG*5UZa8#mw{FD)KkH0#Zv$5BZM}LKCq+LoCMrptJn2Jo@;* zr{5C6UDQtKTPejy!mo&C9*c|+?f}@KRwOhDBF3a3Tow4QGfM(#3H=V-J(6KzW{OEj zY{k03J_ZTICQ29QT>U(Fi7e!T1rEUb^_#(A2F3>t+Qaiw@KYv&`t7Wn{@!??0yd47 z%**Rh=IL9{z;MQ!$Spjhae3*^^Eqa^GFiXSa2huyUH+m}!S$(3ojEr99&b661h1#*&5K#+?unk-pz5sP^LH$OqN zegFOSzaF;#<>>z3xBd6g@g%*ckg!GgnqFw7p2x7o(Hp_4hnM zL|3^T{u(=s{(cYd-hEd(iQ^EF2LOd8et1Nz)cOSq|BC?1#hj*zkD3zUHL^=?LN6~( zw_zbl9Ws#1RW!PW#tdpe<1EEA$og=;tBJwrOs&J3Zk$wfHI+j_4L&_$%9ca}O%;X=EOxx;gH-(9k`kaw(vlnT0e9 zP(PrnOIWw^c`N4uj~=fr6om>z4#`=J4T|@}AQq&-7L0JOk?+a-hW(8`chs>-5!xtz zkA@<`G=eLC{z1MMdBXIsU&Qc@9Ck?Q!UdPmm4T+*1MPhiC@Wb<|18|hAp@D*u*gWH zGk%r3Gk&|`z&F$`xUksZDvE-u&8ojxFXHQ2eaF8aQYk=+{#2j#c*_fmtrU@oo)+|| z!DdGYwFGfQwvyE%`qv&oA>UF$&SU!xFR#UJGJ+l_k%keQR@9Hrwt;O%Ug?`Smw1lm zyUo_t?>{3oGCWxEX;wUT)sl#r)KE3LneZ%e17S^?Z(aMpxY}g3g@#*I%;lVXoN@BI zK5Os0^f%1|<|S9p?Pl3X`y=fXQtG}7i|$woyF9CrwdpEkW5 z#{Z)sQ-zN1Mc#{`YCTn13#Zn}gg0*=gCrS@}_?pvB@$ZzA)L&QA9hJs)RDmVTz-u`6(6NGl7sD*mGx zC!%@u_Tst5+QM3Kees>ei|Q6hj~-ei>HED{_2#=3aPmKy^{4lJTzYg?$sU~OBpszS6yrwCDJMpjzMtd_1> zs>s>>sN1-%rtkZP5AAg(5>qZ37sJQn^{h46ZM;X%xR^1GiEZN3>2|KrV}17H*wjJ) z2bAYO#otxvGwT$(vNYEZ%sPf8SKQpF`Lq&8|QgF_qX>K?Gd2#6SL}8ygYa=%&JFX zs?1K3|MBlAm(gFmS!#J4;qmp-Kf9Lxs+tQLa0yFpVo?;WQ+T&ryeno>Q&Rg0-}38k zo4z?~{#efGcmF<(+x$s%1IO}s*_7WYzg01mjaTD36Lx!*Ds2EuOy(}RFwW#k6$9D$ z_d1snw1ak3w&kPvP~pPas5G{=;$F(hYOGptMWHD0ZGL8x!}g*pZR~UiizSrd`1#AX zGYZIAfB!jkZ`bi?Mx}&?I-?~Gn!6hwh2DK@?4$N8I6p;ns$z|_suEy30HKd*&#j_hn=7D&CjEO&IZ{dv(T z!I|`5sTY)*(o&w!__tLSp2}?+@ZCDqF_jzpruq0;hMOP0tyjk3K(^UK#mb>!k%N%e zL-_*~w0!mK8J3Grd&{Pz=g<5z&6-nXVSKGuVj=NUG+kIj{Mh2*UHX^P{bFh!JfO3# z^e(DVo&7WOy{VI&9V!RPoav7%+)4%W5064un%W~C(}((;h64X&J0n3n1J5kyQII@w z=AVyYDfUiw`TWcrhPiHs6K@=*<5@1_Z&IikeM2GnhC;Ob57#bL$+`If`cKrUj=(*%4!`u_JKV8u(44EA{FU2=& zA~&n^STN*s@%VJA_iXk=#en9Ne*3s=>HYkfD(_Wq4Wz$w=$LFV=B2^)9X)y!i+2(F zA^K@g=4Zrq0j`6KT+h!g4qnf%*X*NpRAq;z#5S(<3xfTjl>@Us-+c|zUiQ++{v14o zUE)=Z-*N!*^Dx_-cyTnNces}0O5Z(!U2DiJ0Z_J-I*+rs27f_ds-C);jD${}Vf)Cl z{$Q7;JLOoz&gv#xc5dvSeexj0$17ezQlV1z(diq)F)Nrgj6WW)l0F>tE&t$cZrAZS zem}l`!4@8UrMCl0Z>{*%_)chLidxdNPgrhcstJE_%Y2xtAj3neCODey{N@c;Ez>+a zWS>}Fz z*X8`PN`%^H;YXE%h92}nJfu5tG;SL)3qk$+;%d(#Qgi+tt+Ji_d04!n*7TSUYn48G z)2rQf;B!II`jBC`H>ss;n0Or~-W6_I9M@H;>SUi8>&Fw}$Hx);F4Avis>8dPd5Br* z?S`?dQxFfDKMr!xs?$61kE7G5Nq)vkKQld9kEM_^t~6pzBih; ziIp}oip!cm5>`D^cCcE>?@_$_VLBs!p~z~HbJsQf{ZB;H8{PMdDlI%Rbo4~6tnsrI zJRz+(=~X7VI?OHNH}k0i7fGLDVb$V|bw{P7PU>>-e(e|R&jWycH1*n_439-Mt8K4Z zWP11)F0x&b8=&te@#-ai^T`8)8fsLMBEj@w#Imqyw^g>)P1zJ~5vDqOnGBrtxn5a$Ph=#+Ved`XQ3VnUAz(NU zg6uF(@Vz?(qUVlNPrLz1(V!&*j2xvbvEU>Ad+2hX-RnS-E6&;D3`f-y*82P1eDQSE z&|cd5t!tiKE0u9cZDgSd)bvQ2V|VnMDV^FCD60NjbI0lyx*MZW8C;)w)5B!>wGL%h z+Q?{3X&DJCaqC~TwG7}_Yv<8V``|p6x_NB+51r zY-Z9=HQ=A}?q^#z2h#?47tE$UeK(mtXM5<1{lVW@qJmp9Xa-%H=J@^F8WcZ;J33pQ z=F<0jRUYf!X}aiCwt)YEClyjt=^uZiZPkRLdKn65ibC%}>q^Jl`&++(nm`Od&ce(H z%C6NE^nPz}SluGdCGd8vmC?qXa~*xB@M`pN{#yUd9wuv2FIY_X%H}9n=5PGS*3lDp z)iXM5>t;sZtLat`T$*;%3aF|Z+V(|` zID1>=YF@v`F4dD4-FVfbq-7J<<*ctC15v#uEC1L22K5Vk+)S%99xh3 zRGEqhS8=E9XF8H!ZWr}9FH!5;^yuWyua|Oi{7YV)T)?YCm(V~V5O?RHusQG%{iknu1Uj@*FWfbts#nftc$+&&C zX=d%BZwJQbGVHpPPY3*DDGwhVdbjo==W6C{%ig_PqM*dEDvmqv)+USU3qF-vcGZbV z7ql2l-I$rS9g+9jCS%Dy&aiC9r=ii`%T~6jEna*=-DcXQrqyGzK%`74XN)~|dh*bHM4#PZE8JJKePA`DHTOz7w2T=N)f78=Y44w*M zF`Ain_gy~zjPBH!_4Q?Bmxvwt6a7g_IPINuR$N<@JL@9K?fcsF#q?lK2}|m8^YV_r zWp@~iwA4ujKjarGf<)xbI>5?RQ7>`b+^vUq+)V-twHj za4h7GLMZQaSL%)J1{J*A@85sqVyom+vxxsNoF}&Qv9;x`?$T7>w7%juhxe;&Kd2Dc zw>5Cn`uLWkUcFJT8CeIEVp@)BDG3erpQNQ=(6t-R!XT|pyhU|?YPL$?7=Q3czjR5+ zAGlDc<3tB?L{zd)u{N^q;B7X8bu!>7D+4y;OMyuMr|v&LLh=lU54439=6}?&a%^u` z6)JjWT-vBqJ^jcysQ!ea|Kne2+tDH4!YNUAwVcgv5HiREU&I0ROp38fxfI;Fb4OBp z-#16_*R!xlQc+tC!}A9YqRMb+teDYVOfj^D@&~7n9}fuBNlN;&&fp&=KB3~Xtv|Q( z@*XIXlfB&fO`R`5T#MHx_rSd{e#U@+%X7~cB^C#|UC0^gH(&bgeNTewb~T>m?-g#X zR+qK*?2RsA?4kSK@PsKswHked-Tn|d%4#ky zMNsf2q8+TRVn9){?(JWiCd@uG&Q;DD9{lOr`T6@Z_Th2AuL?BaZ_u?~zlmoa4@#Ti zqzDL~KbdQvw|}`QLq}1wai3jFmMxVx(9HN9*ZDAA&GzvI(<{bTTg(!|HpdRqm4DT? z7%OgOYjJTbyS}xGKfj!Bjf*4w>)VpA*FK~3n(0ryztgv*QFus!2+*O8KY>!Rr$&nf4h|Z@vwQNq38uLvrui?I z%Bf%WwY8d7Mm|mKnvQMi$1{^V)f5wil`T)CpI(33M(!?aL&WLD>sc35Zg)_1wu!>S1)4 zmsNn4Qj~kIU|>M*s$wrGdLgmdebNj}Y$=+7da8+*-dvk6tXPw=WefY&7RK59J~}V0 z_iI;|9op|Z$I_!@t^aUc)Yg8w43}6M$IC_Actm+W_U^#_VZNJgu8?>-JeQ=MV)fP3 zF7~F5*mp7xw8pmqA>+<$3`5|Sh~GJfLHZc>$hf+fPk4F znqKQQE{B}a__@8!UQ9jaWFLcj@tsv`J}ebk$H>Zc1#YS+=&P+-t*YoUJf_B~+5XRzNcA&W~{#mosXvr6`J4ZxPjDYA$T?yo8B%VwEsO zExObGMa@>`_Os-V(Gz+SuYS+Nd7yb$VPXJff$M zS(Ll_+_vVH{G8Lw`{zfZVvld%+~>3{UE-Xg^^yN%@$ra0=)k1HOS+reBFJbw=pBpb z<%cUOR}5H&&z5Xoet-G!hqQZ#we~AipLz7S0T|rF+t+C+H-pfWxi2@z@;N=b)@=P; z{r1Bt)+s)|;zRcGF=TLDEJ8wOC(LiIU>HhER4n98(Ah2Xh4pEiL0H9S@MFkJ_Vs)B z=tl(x>2Fe5Q)7MyB}8O)XMV~_e1Ec5=TAYK%9ZH6+cpLn-&QT9+;rbaFQy_sTCO-A za84<~{ed9khT!pqJ<-h4q+{Il(i znXl?c9xD3o$;TvUNON#!Q9Gqnt?>MgkL709NxA-SI?-IO^*?!CnZGKA{)+PkZ6Ud5 zmVIA6tlKHGyQNTiPUGhm#z4lvkrCIT8AF@Y4;|&ZX(%_T3lTIF$1%&?jtU+Ub}wRn z#Kk>5v2?ou?=B18oU}tyW?~UqM=X<5G&nDAWmE9>QM!m|qsO&jNNje67u}JEui46V z+8Nj{sULc%)iYPgTYfk}W^zf{;r$yf<;;9#Tq9~=!0z8+jzzg4v3U3V1yaQ-=hc$# zTOBd3NfEEuRU?_0YNnEVUpv27gx*xpuVWKM(aB)4NZ-P`BCBSgea(p11i*{5g2Z>_DFCZ_6KnzfIU|y?%)fC$c{3ia0)^mC$xO%eiF(E^Zwy z3&lYvre)6dj=$lJbIAS&XX4y@=L-3wTH-NTzF>-kdGY~+oG`o|^Kv&&X_dPehMkIC!LqS*{f=C5Q-&LBn5V1xc(w2oB$BRP|R zC;ln&j!a>bHp4rbRE=d5_!F$|lc?d7DEvd_97Qn%`vstL1Ma?%MMa9DjmMIr zn;JDo{#j|Rmb$+%;K75_*Sj!R=xj0f-0tOen47gcjz{3O=1x=!`B}!xYmt!F5Fdr5 z8@yjt#qv)hL+*_jUYjSs*W5N-hZT@d`);^BJKI>zSM}lv&vh5MHe52~VxJ!d4wV-j z=G9#yItQe_4Ec`~cg;E5D1J#e9LUPE|IhTvaAs!agFg!q!mjFJMb>zH7^wG^E4MrO z_U-P|m$aXc-%$uuS9$g^>NLjRsQtR>hhJi!MH^50p6Lj43V6)Ot2q&zBWNsTrFPpV&l_G4<0osoIY*( zl-wr!n-Q*s-;&i(Qx2-Ps?Uu^K5eAhu!8BYL7MFnc>TvX|dhimVdPP>0H#r$BB%hni4H`wf12% zMppL3mRo1bZkF@zkUVU$*24H~Zsca$+5PzKhxg_d!8^y57@P zGs(DqNJ-_|!zeE7!fY0Q!uz620ynIWENwYDU|^{50A~a9O+X4oUeeuGxmpfRKl&W3 zh8|m1I-hL5MSo8$e+b6M?hZyuBSRXGYtb2;{bKlc`P!ApPg)WZc@o}G>zNh&Im z|DuqvnJIpHkPfSwkDvEE(mVy$9k~&I*e9!-TQqxI8|t#Zs*MIDMr;g^detgKVitpSAeo5?J&1ja%S) z$V>WBy1^Z7jfp!Ozi3&%vGUh@pt?2XrtzC*6`yIA1V)|XNGIa1L>CpvVxcZsGJAKa z^xMZp#`ikIa(gmA9TNMT_vOLLhM(@n36C}<$7Qr!#)y)an9j-IGnnx4OY@mR$3uI} zyhbn#iF(Pt#%JRrx!CUS5XG8}yN2aPcX>`LPws1Nt98ZXv#bZ!7T;O&3TzT^!6!^< z7rt$?tKx8*oJr%4ne5mDS8C-PRV1Egi^t%PCtgc0cJmW4ne05XTBspqT_O$J$k6a= z?7Xcf%ojfGrqpYjZzDFBCc3#bV$!SGq@;(w?W(><_LlaP=o?2=|KXJwXx483tXcT} zH~md!jEU042pZ|5*nlS91Z3Tme0`z0qo%ZSFK} z){ORc_r{DX@|ucQlij^V7`HdJhTNX`<9B!dv#uTxQ|MRFc9% zBt(8RKYx^GlhCw9TOhXGzNXosc$R6;_!hop$=00qWOb&(1iRqk#6oe$+TRY74~8#WhrefOvDRIhiPL7sT*;*{ z^VRzfh6uQP%d^#+y17bd;p5$*j6oZXJ8SIQ9IO*p|HIZW?2#y9SRR^@cQ|vaX7%Yu z@dk}S#`o)Gs~11F3I#53GF}Tx<`AK0) z7{9T0N&n%6pP;jr5V9L?lutDtt;ub3G5xLK>Kt9+(~xplg8qfBTi!EU)xxI^D>A-E zLNnsF9(xeOy3R*MC;4*>)sP5Ad)b|te7ebmloxn$=xzwwI83&*KONt*BusLC41E58 zaAW8kmfzM9b564>%2uR}ho7rvnIb;nz6=-V-Z`P_nECtYlOsW!*1y_=AtA3CBg21A zIv4hs91!^$Z&gI@*DNuoF)1=*!xm

tFB|cUJQ0AQU{oQdgRHe$R2}@3~;>JpQS2 z1KX>)l?ywMn;q-wD@tB#B;w+d29t_X>=mrC;+qbW_^ z)Y?oeUFKeu!=ZLxvop6d`jd)gd-&HRs?TUAgq=NyXk2;Wld=THU0_T{gL1E5*_9%! zZ}XsUVu~|tm(kyma#iZx^*&~cPQv!zL7^X-UF}Nj^3uNr-myHSdX>zhYb98S5g7)q zK~19v|GBSv$EU$-O8cVn-|IMQHq$d{JgJ?SF`Gv=-VVMl)qDF7IMpFyQL*!%{XSl* zL*E*$o|7H1eBt7g{7_ST6?Kjcx1D?1p^={@U>II6_@QRnrlftf`r~1<2CkSJ!#f&P zlczQBtiC52xfh=+Kwc}YBp#&xbf$cTWI9O`^`a9tieTLFa?=o8DKt49T0YtMD>Xpp zkxoh~^VL@rz4^hjIoY|5<>ZN9%_j3tr86|W$?t&Fi^8Rf^B<*jpVc(``YDma+uW0Kim-IceE-GLH>t$ukWAd7=|Do4z^C>_=#I^a4W!RSW z?9s=EwWB`qyPVWAv70m#$9qEeApu+hYpQ+maScQFZOl$rM~TjVni`AQVmlk59sXng z7jaCck#FYr5^ZJ8Ncxi9*v?!3%1Z}tT(p$@zAQ?sBCYfizpfAYHOsm(d65mrRe&nP!&sC%a8}tIRVZ=odcC zTmP$MWb#$Tnu#9*SHgctdUZq=`TQDgtkhA>2)MK(3vH$T_P`r&6`SkpCfJ~87U%k=$6vBLY}QQ1;eBseGtr5=oEv`xU(I9|juU4sAUpUgfvhKWSy6PTUn**XXBl+pH?FY+eeJ3lP)$ zWY$!rGv%O_xNb}QC)%9gjyI!!m*RD!cb(gZe<4nU#`HEgT^UqSN99p z3Edb5vJ_*3e>8f4dF_V6Leejow zyv}SF#shoPl5{sn;_GzyAB7{}PTG&u}Lw(ykMSDqhVOPy@zl1@)S25F;4f~c=07ZO>iidxE%nj&X|3ZDJqPKmRxbSR z%a>nlM~3WAjSR>Og#Rd*N^v+fW&NW3Db7ov=;RT0_LNN(e{T3g|C|0NGD~ISK(Uo9 ze?R)zb@%UP9xvwCrWZ&G=rC1b_irxirbNaloDX^FR}EJBpJu`5U;a&G+_r=ufh6<2 zISXmRZr0}~6dyc(IDaRRckr+eW?gDlKJcyew4BR@8T}4R3%&pPMb%|$mtO`iE=bv3 z;2a4o#8C zqPb)re2L53ay7T(m(wdzADy)7kzcp86%9VI-JSm!<8O&J$26| zy>z6_YwCQ`ex7gte(vQK#(<4+CCk_{dG!37Cqs+-xeq@0B!&;Bv15DXJ>;Ss|KyKp zd~=pYt(Jl1-~IiXki~~X9TOdjycR*%w7PzLGMk^ZFR!#D^HUsRPQN~=p`9VxuXFHY zrTgDaeYy~P#DS?2tGpkNO|Uzqjl9w-xJ0hrRMuRx<~^v-?zi z@1{QkP()t+mifYee)s&+uilAOIXC1~=2`x*flOHLEqGp`!T79tp;(Duv1Dcj%5=fv zzXQ4LSZ-Oa!ZmSJZBEvcM_8I{ADW6jK1SA%+ckPbxzX{nJ$QW|MwfZzS> zJ$OFf>wV`RF2|WYdp~(U>t1W!5bCrXyjTp!sbS%uT5O0CX$85+_|D1%OD_AacKGF` zYpCrIXw8jA!vHYK%InB2Q-hl3`EDgp?VYsw+M{QR*R_V&MTUgAwMo6DpTpZKfmWzdSv08^uyJhFCs@@`{6dWT)8RmILgYylkk5kx^& zY*W-zINhcO`9aQ9AV`HA6;lrU_#J=zJs*N^TfuRP>^!R07K3R$v8)hfsfu!iH`g@PZ0`A{`y_y*FjjOd7tp5n#+bc~B<{*=3j!E6FH9yWO z=5FU*_{@r9{Sf>g8QHCGXUrq-b?SLlR#086gEu_=z8oYc<8qi1F5z}7A>>XJa&Z)p zW1khBwS`_9nJ_I*2jKoz)Pdhu0nj^>haQ~>sVC7VwKeSRrhOTJHJFe7PCi0)cWdnQ zzXlY#l~$(j)9}g$^-oW-mIh3MbIm`w`<| zI#`m3sCf5o5@=VLC!J*y6bcFoDD8#FHX4WR?+)eoJKIbLY7+!oC!;NLi|`O&){;S} z38m__Z0ot7J$)(%Di2B;nlvEwA`yv^8(Tt4lC1D5DlV)56-2>jqocpzBzNTA#wkhst&808N(ey&LP1v64qv{8+!owByMg0AtSZ zqG?YIs~F#|+>xpG5CkeQ;L6!y#&i1gX(q4=2l7e_7y!qCy=e-lK8H(tqc#@Kx`D7P zV92@z_%6nv4GIOODRQaOaT6@(KfYXXRLovtp86;k))ku<-HDDp_Fd*MNp1K}qGnN6 ze!VdtwP_4AUeIs(!_C&%rFC~}0)y`wC=XsU@}E2iHIJF$_iab&z5$nX0hLVwBm4Ou z4m->DY4}>S-HFn)uUS-bNNaw~eV*$*+7=G;U9tD}`uo)Nb1W>9P_#7(Jl5WV2pGIN zQlMr@O>J!pjYGHGw8||ccRuTV9Od?hhlkOKSY2Ip9(>7bklnEW4spg%^0Aco+A}+A z)c8K<{Bxl!nV0f_hStPl@KnDDXzOF4c2*5cT_GZ3mojME?<#=B=9-{9qwC5 zki*x1g6dj$#ez(AIIu0svEI_cF1x*_NCwSrEX3L_dE+1#s@oHA4_G6JJwj^=Jhnn8 z7jv4%L)OpmeaRbtl|j&rz>-tr=exY zJU>1)+5^)GBz5<`U4xnPer|p~*Z5nkjn>TPi}xuS_qwx-i0Hg~_YSxf@<5SqEa4(6 z%QEZZBwuq6Vc4{v&HGL4Y;4Oik(`%ALv-62#?2)gxMDyY$Ua@PMEs-pW~9JTFD@m$ z@Fwa_5N73q-QrY7wm8`8v_R=AM1@%#YZ0f?x+hoZx}%)mrcm>D?aH0bV6~tAU5x8v z&HGcC(Ed&4o){ZN)9o}Gkqo*u;g$X!>%c4_fT30nn3=)dNvdc&y{oCISpXh8*dG?( zF5O1Dyf0$4y*2ax7o={{8>+hvC+AV9S^yRA6c2mQwlFco0~camddfe5Jt<_3Z)} zKu2hr0_zGnnA$LyOh@!%ifkA-M37a z@MZmwdG}@$=}&*H(H@2ZF)=wge@3Q%|NS)K4E9`%Ntu~V6NS7$CTRy1fUI!!4JoNt z^@a+t*a7c91MVWq$h4n$vYNS7Hq7_f+#VO+_S4b|ti~9?y76_&J|!;hs(hueAUMf* zZLIZeM!GgZsZn-`PAJ?A#FPZE)Q&Q;CEL)Q9s2lNcWRH9>ul z38fF^Kv*y1lBQs;tBPww8dc2}r7Rj;R z6gz?ac^JkXMe}IrAouvxeS`BGU!7lw9Xg+02#pfSUXv7g=$JO>I1Lq#c@I4`8 z6W6|Y>uH@(lDl~&4o6t&Il>*!%!1=*N`C%#{V8x9Y6p$ytMKsD^IC=BbrCWNi17!u z0@Udag+_b8p<6xQDyGVbYPz`mS_Vx5Y>=B5^BmQfu!3&WQ_p@p)t6=s6*Ah z>C!a|P;N8~UMiPDhe~Tr{`2z7izdd>7)-kNFcc_}16?x+VUpp_O**pfAb+PXJ~Szy z$kYf}fL+76`0PCpg1|Wa@|B=O?0ssq9n2mwkvca`yF?wNfvr$Mq7`aDLM z>B0>*rEy@80s{@1syU@DLzDRm|28hj-~}TF5<##P!k{$R7CZHYSXrfz$Q?YVwHVg7 ztwh2hQAtO0*lwyx-&^r?{Ijpv^3hKL^e^Oa4-(ylMvn~zP;Rw>ATlT*AQqfH#@<)a zDnL~oLWX}F2v#dP!ObZV+H)KgUxWq?3JXgC%O0dY2R%_6JfGV-+qT`=gvpBr5hqmI zVAW^CQ!V%3!S;9KYGoi@T$&xRqoJK1+rd(!9d3n^$U z895mM5vDMaKZ*ox@KJSK>$vHBdhFGNH=T%?HX7LUSHCy9?(-C#UwvTiW%m7!w85wy z1-f}MG9&Ya=5GKgB_7p~KJIID4$_3(^63aKcQkvC= zeUkg_S{!(tw2D%D@BIEDD?%FoY=9nIHfRrZ1F9AE97|{1pI@?qTrBfQ<*^J?1urSN z&HcZ_@EIe7eWRsRN^I#zN$@!y&h9NFPw)WPaahpx@sshkb?c zL(_L~qOes37(>rHQ3F5%GLr4MM(gr?sWw4-eDNRbK;`*#QkjbE)`g{$U)zA}JM^>i~!`5VoH?D(nd$_apUF6%b>Yxb;0?eZ$tXaV?{bX=3V_@RC_ zl)cQhK%jL}7YEBGo5*(N56CA(ZVXU~^~UHulmF*?4F37(;&kG`O);K3)km;!nd~{> zt2^_VSnMZm$oMmqCAo<0-@5hq86Njh?ir8E9)V%#?)dKC-%Vfc#^(^o;NouOlAEki z`KM)!Y`++4ZB8wExqj;Z3%@z5?Qk|FbFpmSEyh!9L}6#C-t*_8aM@o|Xl>ZWx9-eX$3$1HMYfs(j$PB2 zU%IgTmsop>J3HQ>BA*VzV}Gq?YS!9R)2;SiFphIp8f3$xE3(5Or5&>`8N&GQ#pB7& z@O|j|Lz|^qKe*|fzwSE2OX^rnoz%5YenL%S+xRD4H}*p-%$|i|w(T|$lgxXf%&Ys# z)^0!UaB9Qz;-XGckOj#jE*}*W&T6@VcOoARYNKC0%`{$z-tAzjdl{efp5+S=6L2|HX^NHpC?y9fcO4dekTrbPv)e{y#H06gbmEMO z;xoDDok46SIL>{T4{*fYZbCO8kH?dm3)D`?m1|%L5>Y3>iuy-|L-W*tI7{0REwh-) zMEaKQ^?JHtyO7HP?!*>fyqm3f0Inf^Pp>HW?Q9(?e$+5;Y_2`&58iWf(48isjBO>1 zkUQnkoBpVJC!WFJJ5g-2eO5neJ*Ul2$m)(HP=hr7tVjo>qsXBx1IA&p0itgctoWz@ri!N`)j{;pU1t4w{e25n!Rfz_8i)c-B$#btADSPI2p`K&aSLWzz=Px|zD*)+F7iFzBN(rxAIw^y{k z)!?@IeIvtkRd%%2FdNTZh+YF$!|V>gQGXM1?7WCR+R~(O0j$PglW7}@IHOJQ5w=zY zRt|nlzvYjfx_oH3=zv=A7bPWtJMf4ZPRIkX)>KaQ%RNQ6`Tw3|I?_3hKwOoK zg_YhY!|r0=i{R^EO&Bgk_SDFtWV|-Uj>d7;KKOv7@}Q4r)fvM#v+m4h5Tg z3;9eiRwd~zUftSsSMoSj-fX;|rSl?g$SmU%+4DHM@~vB%@ohyu&L|ShCCyk*55QOV zc`$VQ<)wDY+n?~|gAwSdTj?>i;|vS4-FT>q(QzBdC19^&mpk8BwC5W$*1Lz!pm*vZ zSD>4N5Bx5LwC0BvzxClz2%PEvWMwwe>XhQEMlV@ZC-hvZ z$2ZnGV883eQr;E6dEq`0SHA7AW-yArAZsjSVX_43L zJM3a06xx>gr$NH<^dkwbKc@C$2~o7#mhXeT=Zu6%D0@Qfl>e_}5^v~KLTys5NKv=n zW>S4rE8zS1Bvpk-p`0B&-1@L8^bcCId7S)7+bmO;saD(M;E6LXQX`EL-=z z@>7y&|D*t^c5BZewm>MY1jB?u2l6HOZD$4m<|=wFq==`-4-t4CVBG+Emx{@u7vIs2ad zsKqj$&+~l*@}3jwTkXE7x!%^MJ^#sQA0_Qk3p_mAcmO})-F(n+qKVGlC_hF+40In` zi)6n}v(LM`A1wx1PV@M7r=Amkg*WNSerPAU&|=p-hN5^|p?OlKcO{1AjbQ7~sq0?E zfWqKJ2^$JL+l<8u!a3GD@m1Cd*O{JK%{GW*k3a^izL%ww2iqu6@9;cxIUUTTl^cK? zx^T{2$~sS_n7uU;HqV`BQGV&sK3ll7F5Sn{{5L7Ju<-;QLA>|P`(8ggi#kX~qgVOp zgfJn7DCR|Mxg0eWw;Nl0$1Ul zX_y9S@zOyzRqk!edpt`!%|xvU-$TmFq`EHD`#tXX#P#dTQ#{b@Z-lAM{N1ahD#vy0 zZ(r6kpJKw;P0UCcLb6TkzV5XH9nXrCRk4eBmcja)s0&OKP6JEdpXnhtytm>^Vs+uE zi34gtax>8=NyFd8!qR;oEE?r}2jNDTa&LzJS;4A?a=v;^zSCyMZ%>pS&NYdm;#@XB={Si=DQ~={#&KW)F{7&yIFfdV1KH6r&G~m~k3VwMylX4rZPn)`j6-ldqNY z@Ou@FMpvW<$U85klRppi&CX1PO;X3OTBWa~6W{N)?}Lbcm!rcMXEDTj-CDh857|1@ zwNXZC{jf{Bd5xwhCzP&t4wL+ZuSeNG4XJ=XF#`-FR(B zm$U`ye)_{OCc+cM)OOXdFwa+eEt~zhor#y)0-dI`bbn{iSr9Kv$VtS5K|NqrAkY%H)WpyEMk|8NxCY!iJb!VdMWgqq235{-J{0MX3=l<^glR^Aip{o17@cy^wk4253ug8i;8eQgUT%MA>_?aCCICU8qu#zR?^sHC_$D47NwYoV?v%vozki4by_|^F4eYSJa4Gjj}G(bHe0n ziE_i&NjD>fZl+m9#FE#5*WUEMd#@}zGQ%AOXDZTtprc`SzkLz6d7A~S2hA~I&a+SP zMe4BnM!`yy_PpJ4`b8}Cft?<@@aP%Vj+nOvm9!mNDqMe-KRep6`4EPpcWrDYYPs+5 zZF(oVxD7M2ok?ABy;reASm)MRJhOpnqV#{aZ;UckbC)QqL9PaSAdpP{`zYIHNu^&} zkwb(TPh=o^=0rY()B*nL7AoZO0qQ=)fd}Z~bdLs9O5@Kko$kz)gla!;(SLwRY7(R| zlA$K}E^aSM7=3Pe_YxR9Y*l&jJ(8?02kQMGNXyB|2|iW|rA|`-k)JOdL7*BKnw?{3 zmxX#$ir_%_>eVa73y-(abp@7GnKqyQ>LoXo0y6%mmJwz%ZSI7Gqz<)*AG0cu!4 zhV9?o)cwmFrTsJ7^T0{Ijh~-m5I_zYxW&or#+v~fklb>)#CnX^A{-n5(y6o^FFW+u z1w-CQG2K*n>ASvqnR^L%!=?g44vvWSX9=3;gQ1qEPE6m9x{I)}irH&XX)|E^!$&jl zrV8cIdjJ`b9Pn@r6d5h*u^@*c(5M1Tq~pL};C7X$Nnl}NX<^Ijjs>$yNQlcp`zhSK zS@U&~NOVopDr!I5xBr%0*7Gd*_bBk-+aFc*1$l!IMo;Z;N$orQuK+ouQvknN2KYRVkZkMl?#-Ga-<<2z)Pi z9j=2>wO^u4=q%qy!|X8t5N zQSX~Lv~mU10FL!DV^+TS^ndis@1!wQLVe0OR>P73bki~q4~gciJGC`-C1WiKDLFZc zkZmTA^LF35aQX)oM8NG+{3HHagmV9e$%!(TqtiI9SZU_6@%^m*l0sS2FH@=;&Kjc8M?VxX zk^S&&q~jGZN+y6;qhf*#4dDB-P`|Gga%}f1DOHRi%MA>SU)S#Rhii$fI*ak#ZDO>i zL@`FcEdzfOI6KozKr(u+(EYaj&(BUq0U;shBs$;gdn4;#Esu3eks(SdsuU=j3#QCb zSY87PMs?iH$knUZow52#Mh0A{ol!dL5);hWR!tV#(@M;rFt{dD4meBI+Aar^iEm z4ag!sfD3GYGbSJz6vSu;w#b?Uuo@0yjIaqSq3F7mMXuBDs9&Ql*8V}8k_1SkS`Bo3 z1S8x7rXc)889K&D8v6WqG?Fbp@4O{noiexN#qbPIzax+dgbf0vm7v;pA|zHC_054< z1RHG-EHvCO<$ZIZw4WA?)@G<-b!iTggekxp>ddHpnyG-)`MsbwJZ~1_Ci&;-mkeZCm}O* z6MkdPxdckexB;^SYQGj& z73ei>zfd??aX*hVEf};(e?#7Y%4myi0UG{Xrf@z^2=W9_{k+Cj9g|w8^R4KIgz$WI3F+@b;ALO9ZAB%03e2>0Bl+0K3X*vSHQ*mK~^q^pFd7+Ns_W z4XAkq)dz#XyqwAVRNSU44aXPu?}pWpCtH>E?63v*2(nCoet=ToupcWyon$Si z;h>vPdL%pwQs!kFix~;Uk0y_Xy&P0Wk}zl0?G*PX@|bDN)(e=EPjjVYl23 zmJHL)$$42k^Rm~DVKZ+M0vzfKx33m7xJ|yVYH*Z+xdVw)o6=lef2WCNA)l1b;^P5F zxk(WJ>>e;~QG!Ld(NWvRChLb1MK4$l%R2PlB`yf+=5AX2o&hSpa$kx1-+F)JuR`7jOR5`Qs-{+w{eyn-10c~lRw09wt9s^> zZTiQtT>=BDBwAFC>+t>piawh$eiYX|2!KlaOs=xJVJKm-tFzk>CtuDQ0Bu`0*L+Ww z(k;yqe<4RDmeXIZs2pN$`TUzosK z>H&fGMCq^488^oaJTmHUUD42Zlo}-H^K%W*>QzZG!LY4^BBMoK*TgiUeP-iPc7N+J zW6^4mX&)3@>Ry~uOtDGD{{ukJUz3L#o$HGTV-|QU%LfP0!vV0UDNk3=R-ZRjQ^4>< zoOA9u6T~OEFwn?OJ(naFPlByweDCjmbQ0PwW4VXMB&C~FLhWfC)cH~M5gY9?oEu3X%T5Q(8 zP1-FtkjYA#4a)udXdAouzIC;&YFqAqQ_wPw5QqE(bJ z=!U0^FzP?UV9%Qq2Rb~a3vVo@oK_`en97#{1s?{=tv9Wp0S*!J3ED$u#E>skBQ6Nw z+$>3rRs#~FduMKgzQBnrWNI*VpG)awnh6dopA)e+7QNxp8zX#g*AlYb#N_{ey~XAO$f9YEI0}SgiP9^lP$h z5AW~ZnG3EAk z3i8!QA1Ur>+0HyfT2QjXds%y8^q~9%tfG&ADjh{l&2Vif zRr5TQ!SCgpqOXJBEdqaQsaD~S;yI5>AfPY65pA}5ZE+$2YBd60&=hOv>2Z#kh0_4C zTfZkm8%a4t%G&1pUJ6ipQx5#8fO(sm{(vmdoz7wa&FUHhj6Pt} z04rNrWgF|DeUZ{yeWw?(jF+flAEqO`gw`g{>`$^hdzt6k-<`<9v=w+h!PC-hVM4s- zTs6l~kBI|G9IJQ<#*?C9M#rdiHwXuTV*MwX(C~!m)PW9tEjk4>KFm?GMh~Fz@!oO% z^6;=2BcR*MRX+Ff!5;-ZPJ^y@Mo0L$WWh*;o4F35* zOSdnu9m|W^Q$B1z@jNQ;#GJ-?6Ehqd^khE?HutGE63k!*r$kdO-$FE~g}3t<$|jzTkaj0U zF0X$)`yh#9w^_}XZ{;>WLCQ3iBJq-yu+6@kN+UKd0NPn5A~M8Q{^bM?PIgx|zQI?C zI=!)j9$%< zH*euc@)Ia*&wPB8A^HJ;Km{Ra7cLaLAO1KIKV!jZ5NHt|2eNq5vhUgmbf=q=l8eAT zoB)!Xf`r6Vf-MvWWObikVQoDiO*X zbe6c}F(v_3ZXEdQ^78V)H%^2w3(lAB%|Mvrc3rUov-5F(aoYF!+_&Bf_>P%XzwUnj z8Bdw%($;P+IY8YKDDxZosiI;L4kzPK1zJuuWc2K0EfA}bb^zFREl`XUMZ>00H>drG zSKoJc4)JLbgGNn4L%?}-Lu7qUKV!Uepq4JG`Ug&t1rVU#PY|w<5b1mq zp(#KBy?t3Ddcf_&$i)n&;MVh`sc74gc7C>9!%`vuty&P)H*?g{OTgg)s((joJO0=u z(F~Aded=BjZrJ=8OchUkBPITH_@TOH>FUUdjwyRLC@G}veVh>%DhWEb1%Cv41_b;A z9qV$2cb97gO!W^EdB^8uce~xjrxQo;yB*|UW!?DJ#dbq~~9OV}-!rYNOC{IOAPd*GtoP zhYT!MX1LT_n!Bao0a&@~>w{cKL=g}N#|Rc?=J6M!;^N{6!J*Z8-o)h=WF0+>qGmk0 zoP9?jvI)oa;bDO?sLTRf;T9+#6$C3!g`ww+dBgYAvm?TK)p^}z*LZ|m(HWC|r>a1c zU!*%RXd7@*&o2|g`IbKu{$1eHI{$J_FPmY;LC>6o5xb6=MbW_Vj~j^y(3Fc>++N24 zl1J_N{mGiTcwm?51?>X@o`5(gJZs}24T20tkhgT5jp*O}QrDFlz!@yQ)UT8Vrg;m# z*onP-kE6ZkE%17232>7<;Pm^HYc*O5MSo^4Sb9P&Yv16^;Iiw$jISd6n*;uoKg%Q? zoDIJIB{$x}vqf;xfrIXqHk!a04g+@0r3WTo=4+(UIE#7cn{z7Kb#>lej3?i8$K!~~ zhivG5(9FNKXO)ghUv2gCHRk<3$ zJJT#yFL<&2MAq{#LMEO9EN4YNncCXD{BGue1Tg25_X|E>Vif@S?F$VQor+6f zJXuYReK4B2JA4-(EJzwO&!VJHPzqNv!}gNvb5m$agPS_#)-!|EQ)&rB$nS0B!= z0MkWaumDXO82q{LW*^lWgiWp(Rt!Q<&YxIu>!5KxU9|YiC>BUGWk)w^6*r4Lc z`>Vd69daF0m%=Cfb4~rswtfW;xB$qqBZMB#$3~YtpbioO`g`4$wJ18bjgC$;IZTXa zr1K0hU0^CKXsaik05@;LP6-rr1kksoswqGjERn%gsO5MIHlJK5Lrf#=`u&mD?4E4G zF>fsat{d-3U1Wxkay`8mDESrr^XEkm^>wck7JiIwkdc5%A0BKjo`)*(F!PMHf&P>0MVWfvkw;|EwejXOWj-wG|vN0(v5xxnIrJY$UKoxV}4KKL517} zns`7Q6xVw_K|CWJdQMmvIjX8a7{;EmZFKm7!+FR!3H(%0dX`Q)JIu-+Wbv2>BK*w& zC`KnYoxyKC4g6@==D#+E!D(*EV93d!L?eMjQPBNh>t^o(@=wY>tBhRGBSVGmHc`QQ`er)z&9hUSee@!4<3G@UP7qx zTThuOJBA&dJw-C_MKka$t(firQDDtF_t}P*&!5jw@{AZ6}u63vjof=fVcrR5d%oq0MA2976rlY)%pF#e$@ zWy4a(Wf>VCD9iCLq^v-Uw30<_)&6=~BU$5YuSvBC%|a3$DVC#4S^;7M;vr3>P6DX6 z!Epn63F*-udwWmBK0nVH!;qR|=kXeRMtxIl2Te%bq8g);p&by1B#FCr?J~Lq`%Ygl z2!5a>)vJ)|TbGEEV5(9wlK0`lL4Is~Vx={{ar0(#S|07@eRIeibwV*Srp6xFz%5Ew z4JDdE8RdBH@9o-eaytETO>b>D#AB_#T?g`kZTZ&vG*Ao>Ru?5by&p`C5xka>kscdP z63q_}2^%Y4i`G0kt^Jd`Z?#_rC)SDfTd8XGL+NhhH4UVHWrXRWbP|;8w6)QwhRR{a z;QH(gw++yK8WdQ96q$BbK6jm8kj|vFW|>LxBv05iETAS+-n)N)3&^740EIMxT1p6j zg2+I4@#i-&*l3y0pKpik8<*avoo=hc!4UQr{vJ=^HmE+FlCtCMR9$z{bCJ^Tew!-^ zbV}?=k*kkUd(exj3O9kVdn}}m?m{7|PLNP)yDch}z*n^!&EXedtj9s}E2R4$LqTvD zDh4%MszaC}DfCZmMQ9t`ej$`M4to7M9`0}9)o3q(k4SuHpifWhKEQ6f8LM)%1akb% z&cUG?SnRa@#k-7fz8Dzj2-_8gI#i339SY!EdzOGQO0bL|B?ROl6`}SLtJ%KnI*l-k z>U1+50x8_^Xv)wLfh;Y8g5_ZiJVHzBZj%OWf>e3jMhz$30Mn#xq$Fculhttt27^O} zD`5y@dmeUV?Th*G2iP>mMnF1&dB}9;%$EVf%umI3kvqu24(bS4y^t0+ zkwp1-&o+J&%EmK1)>9an&Yl&ABCpS$Jju*IuKP+OJVi!dz)4H;sl+!Sn&}IeHw4*E z+OqF4AHx6=_Bwv06$J3uin(a4+(SCZdIb6Tfu9Aj{qc;>Pj0%4x-Sl1au&4@>l1#B z0Y_y&c!89B)~qx5PWA44{Uka+JqOwq@4mU~<+>3yG7;b$j1#QLwo|8atmZbFJS$VRZ z!jLmn@foJPJ6I~^o9bk8dh68pst>G^?521L)N>WSG|F+Onjc(87dia>n6anM>SThs zSE_7R=N16MC)l_#o63;=HU8E2BY5Z|+7DC=_hD~42WGAM7@WU%)RLfmX`Is+B@r3>H zq)J%+QF90x8-Nc>(8)lOCBWYE!0#{fx96xDwHN$Y0Q%)IM6S9(!N=L!XQHYu7~s7qqhSYr!3R4mcduwyC+`YtMwLjmoN9aB`>Ez zTJi_^n`Y1O#g^U?`!taItE{W8E3uj2=&6+nuL7-0hXg>DQ8P(D%>-o+g z);}MuD5W#3rk$S4q*k^t#&er{Z4Pji+gy7m13te_28$_QG`5I$Mf#7cxI751VjIpg zgnEo~LdF2xK2-~lX^6G627&(vdU<-Rw#11X&zcC+VJ~_qZie3L-%rfhRh!o1);h@7 ztfyiYSFtsquR}D23Bl}Lg| zzP*0I#%%BGPXE@#OsVqvXLTG-zOrKnd8VRVk^%++r`24-Sh|3Kz8x!)ChvF{K}X`3 zJvs`%o|N;j!z&e(!CSoDWA)#8Doi&p(lox;dqX>!fJ+Q*6%Zn>@qJVY|)?CW1ul^R8fWq-s*pE9E;_$Ygz#8xJFA7r^+fc;unR0UrDe{OAn&^|uAr zXsf22dT>x?mI9M3d|2E_!zZj~6A)@y?HCi92KjPtre4G1eqdACf(nyd?TWzzYL&Yr zC}giRxCZz|MP+yp2kF2xJ&c;iED^93V~BcczJBGOF{KlBRRw2$B{*cCFIi}wXFma^ z`{#$Lf}9ML27rQyt#+Q@AmOnvDzd4DQWUVPWBFb%2(T&89Ce#2upMRQ=2n7qhx2$+ z0&_GQ^D}Te+mj;9F=V;mUV$Gq4i^GAlz9^jo6eW6+y-sln862-y8;{`1;xt=P`eOa z*tT?I$DvEn@a~O(fq{XBB1;ZtCfk~I?^Ne^$0Q4}K{G232R$XI)(bi|7fwk@$rjkc zxt-<=P;_a-*D&?%=H<)xk=78@`H}&U<7J7v$owUEqHH0w@7wf2QUGAQzk`N)x@1`! z=>#EwCd5eu=D4ldLyd%Ma4=L>R}Y00aJPqO1RVYmiw7AEP=4oInY{GMv^x3owJoZg z?6qsx3V(e$fn;^S2Dtm<(_?m)dJBQ0E?B7`L=l2aBZT{$?n;i{(+ueO)1bgILO!n%5!~5`f8&3`D|qbjn(XF7;&s@$^+Pbdz!gz^C2etaV>^-V`16^!MW7pJ;ZfSK`2 zH3j%nyAPf`ClShdr$$X@4d9h_I5jLnNwgZn>k+( zQah3pj<=;uK%EbDh)UJ82xIq5plr3t_skf8TMG6~Jb(7XzKC)&;MeNsV1kp!E=>Mj zo(CKUJ;>Jw#6gQVY0Z3l>s{qEjO6qskl}S6d1KTH$?bWT<3r$s&ojB`Dq>tK0Lvvl zs1sU*1T(v)#yfLU;9HAGM~M{Sd+1zskG=BD1;aJUr96%dQ*3&YqoF4naK#3S{j4oZ z27sh-&~E*zJJe^h(a8qB45*9~01SyQBM!5VcBerRTJ3q@xInxl3%T`CS94UnJ!+Lv zLfqz=()c|mgNX4jYwOEYtTirCx@`Y{%mpve0$K0|(bMbQ4B!n}9q70WL-P7d>UUs6 z@e_bC{!xqj#L4og&F((IQSE|Zo}oiIE}~UFwAs;eR&%2wJ7$HZJ&Xr8ym|inGdrC9 z5U4-r&YkmPye97Av8)GmC>WVUM9R7et`M8%S`S`d2g(?XG``J%%NDgCHN}*RIA>z9 zwMZ413Mq++H<0o@A6gi}L4ejS5+!4=fL==ayTC`kJSmH$(ZV?sOb?lP6YLqTUcY`b z01)xwbSf$;N%?|2_io?>Z2a=|_zAWff(64v1t_>B5I-ldgOPFL#BW1>g~m_yYP3~w zQdcTbSEA#2z;TazsjI7loE-sbC~*Gt=r8zXfeiPLvy2^|MQFQ{+m3qs+F zRn=;4wfwJwf-0S_P)poVIzm7RrVK(X8#hmg76|b#ONbo>E6|l{(soVe&d%sy{1a5& z7)V!BSGS;ATVLPy=r4*(4T3azU2J13f)`s_T7)(nhN}OTkXKXd_&WJ{HRk-3u1}Tq zJ3h!8qu0hDSZ2Wa?{nq{$6vS2dkU2p;OL zKx0{Ei$tAV^Tq_lKKu}#$ymg$62b!9EO?ScnFs+C+rz0{xIlUcI3)MLRAn5XY;@W} zdx~!c0KO6O3JSRfUjqWw;30DX8p+rc{Ppx1h=e2SYSt|3b@N2B%Lnt_m4xxj&(h`f z*oVPZ11!KRLK`cC%;)$vY$stgLe*(*2EZ5NLqhtrV;paLe-WAf1ji@d@VSnU3bI0{ zC8!1NB|>xWQyH`>S|ENUNg(?f7G6T?M8X;#z zs7_iq=p(oQ?ls7!MOo!Df9E}eKmyR4z!*9jkFLoQ{>c*Lx*-piL|jmg9bnus=wK*s z6#!FP)@cK33^Rkz5%6(sw90_^*a_g*-HhYUc&3F$kd7w!6sv)eDK>80Q$02}#{0_3 z_U+#v1Q8INl!5f^B$NjSTl6(GEsvpUf7F8m1K$gpq-5(|YD`O=h72pOS+S!llR~4gRF}16#wX4 z%~#OZ1K})6mOK13EE+R;u9nr=k#+L76=QLrlq}&>)qq}MD)F}O;B{_CRqPtT8u&h0 zbwDl3%MFKx$V4F2lgh(1DmPtUkwQJub51`Js2097^B!0&QYkgXar*K%5D+WS)U(O! zJ#9Iq?~NVi$@)$A*c|$~y4rYR?zhBZTc(W|DSg}k^2w0#8!|Brrbsh|FC|!cfHv2+ z@q}DThcuGCR&+5$ZlcCawuM4Irz$Fr{Mft2{@%MQgU!9(G;-A-J|H%rjt$#a&+*zD zdnO$;a+z)dTh^_!$(74c`yeuWEsnthTqJS62$=|2mrCWHJ=HkKCoIrSI;PPdsgntt z;x0qH09@%C&IVC2RE>eALT4ya@Ep@0nvrqA095n1dH}OZ_Dou$sfQE6ilp}@x4Pa%=@v?TR zJx1gJ7h{$9)|qzaMd(tY4wnO~`aQ7OVxd95CbWJF78*j?0|R&bID2?uWGdXKI&%6~ z&PjW3(!3f4aZ|_{ma^}DZOJE1N$P^ble;|xzxxTb4l|I1OAW{2*Xv7>@*`;Q->u`| z11%_Qz##cl`a&5N-}Q&XQLtrFV1U2r7Sn6+uSBm~ze@XvB@8YZJ z_G6!6S~zK(c(RHGqmqz`R(9Cm3j+|*q&oPesv&7iuA+u!VzqWYc^0)Fexlj=7JRg! zRmOk45@;JY_}`D7vn^EA9OrM+&F6%4c~#1F%8Lnk%o`uT8=y zo1_ccdtD~D#ZRk`=uGN8W{}!&T+^0IVtD>+?|+{PIP`R^na-C;-D+pVKZmn8S`r4vL)NC`va7R$o0*7jX6fD%O{Oh}Onrrfi)1(6xyPoCaW z-RijKP_{&DX;j98WQ&gUvz|;)OGwCPFcy?2*lX0tzS9iEsoADmIl5iwV$Gb5Dl&@F zFFtHd%%hlMHNw=62k(mVv*=|&m}AX3RSp7~ddmZrh`9C{Da=R|=Z zfydzQgBw4wqqL`O-gr5)*iy}K25B|uXCbC>x=tIXTSv#p*o5Pt0hws#3b>VenW=W^ zjZk6VT!B&}vjwO4AJbOp*@poE=$bk9!=fD3=Notol=M?b=e&Lv=!H?AVW;UPzauB~ zw)&M7xfN9Xl4i3cVG!ji@EDD7f_6Ts0%oaOPrF;~ni?}R%Um@SyCLan@lQgjBgd2C z;*d2b&5nmg9**sPx#{hlq*WOSgoT6GylJQh-hTSivmd{Gdh2C&qn3fxWh)3ZESPj{ zq@f1!;Gd6u1B>-Q@M0{)mWJpDFp#5a(bRo10QQ#X)qhEU7t!QJD!EH2e7Bo*rk1Cj&K zAPy}>K7er1%QHJ~_~%jIK`>h+b%HKnQ2k)#*tPXwdtV7i=JM-ji`y^mQXwH|_=k#o zhN!dRzTY5CLr>8U8?(-0%BM=bNJ9>ZnT3_qN9G`h_zdQf0=`D?OyvzOy<5GT=Qv5E z<*9~lpT7r9x#tJ-w#JP?RN8voAYa|l_|J)3^6ffU0A0s#*!x$#ua6}a_OYUA5HYJ3 zNU-0Uno#DfJRxh}Ga;WqB-y}WwC!z&1U(>SyyUt?4!DR`Hl=5)wxUsZi%#-nY9A#Wdj%Cx*>A^ONF#D8h8F3>Wd z#MNvy?bzHC(xfs(*9y|QHDDcnjN;D}dctF5#$&L^(nd1hDK|s64oCJX>)Wij{Dmx< z2xWYD4T4)sNZbsvY;@z5SNT%Yu^T7bULF5J+A9_5d-0_MX}w*?Rki)wNH(NoYc}h_ zv{l@Lrt0`yL*ns%3=*um;17^7_NvLqPyIf~SH{>!T9eTaMWHcuQsvoyVFeeMFR&a+ zU8s&M_<5WJyrW-~qNJjXF#C`#S9Lz=%x z5sv@Y$~)_ns^eL$j}%Eyl8X8}IWw+qYdDHdeeE5z?uK%Jq{)eW;>(x~Q(k5(kn7C1 z_NV)XYQJF?Q(JSB08|LT;%Fo<8aU*Z)so&xLA&BLae+UL zTfUj+(5ELCKCxLPQrmnv(~}bXm9FVmX$PKsd4YPKyE0I}kbkU_D=deO?9sh z2D}vyEw(4;)g8o!wJzsAYCp??#_-S{GYGl&sUx%)U~0O zEr$Io#PiP=g4{-z_*wV--WmGekJ0yiKTt7=!9Qf4-W}G?-lQJs-nonX|2}U>+SxpT z+_wE0MpqcIOJ~b9dAfpFs9uksv8gH!9}Aud97C4|^&>I`&RoUT7LF6>LT@%5hprk& zN0GXOOtvk~jAH8+A3fN8Bt%pvWot0-2H!OZrNO>kE`4ij4(9z@8qXuOOaGa3#-gPc z!s(pwOoEm4*oS-GcTyf7r$a3porraX=I!LIOCAc098+y-D%P|m^-IXh2fK&DJ|C&t zfB5o-@R@7)I(SK`taE*982*EXVKb-M!CJjaqVzL-!^fVy|88;p!E!S%J>#{dDM-n` zf&*_`qTDGbr{VO53Q+t3x9i!_9>(2O_t& z5*G(z+IVsiEH{kxo0LgC!`$P+44M^uJ1CXIq>}ZWZQGb7bB!vXW;de;a zTIEAin@}>#dcwz&gLHQ%n6h?}Kc`}E2X)82g`aXtiMK9;kE256GfF$+iZvp$z%?6w zeK2f{O#2#mc4DCNIeK^m7LC-0)e;CQ2yp5y$W~5O_4jqbBj>qJq=Z4@64X`zTo>LO zi*0JU1(3z%2tfygTiX_=fq$J2WRd6Q=H%fg5lgZS@Nk|mX=IiR7@HkXDP6lyG-gh{ zvW>a%u$f%y3D%_j2cH>S7K0=Q)F5VnM3)fIlMW@S0k)9>7N5KD zw7=2H?FuRxbERi0Oe><;NVJH#ag>V{(lXwM=#*FIh7qr^)+G04Kyh*L0wgt%Nwk1F zF(0(2;cur9#0gwW#i#$AKG~U~m=p`Lnwv->7Lq6UV_P%l*6Qb}`3CSHK#%UNP328M z-r_@^x0jbmbDTt$D(nC2?oFU_T;I3h?7gvflD)SK5s9b_&7^@$(Gx0CM1w@5=1H?{ zQyZm(iY8-^G*OyIl1eHXrIAXSX`=Z(&ZpXY|Nrmzt#|#twcho;>$QH{`f0eI`@Zh$ zI)~#t&f~l#@19v4x+Umd7a?#YnCRl_y6nBI?dRKup^m!`zD>9Pq9SljS!R7wYbv^=y zvPsE26mih+{+va^wdEZ&+8l>0j!pnT^o@J;!N}@~8$*v72{2z_#z_|DUS71C5#Y_m zv|}uqfBn`Wep91bWsT6r`uuB#D^x=NkaL{X+sr-rk7sJVK;}AqOx7#K#r)FzE#Xhb zwKY_XN^S8!#(sUs`XWOvqjhvhazY=BB;y@qgc z@_ZFX`2}{(1rojjCEG&fest{RnEXd$x8vGN@e)VYFPu4R)>?EYU%Ys6t!CVPwXmQJ zdr@>rQl~drIWTb}Kxb+_b6PLizH#kKH3l>I{f?bGww!RcY|1zNud8C#oMl;9Vme!) zx26Sejo}BSfo5x04QGw4rBBdCN+Z1SK_m?gnAe(@m94UgD!A#$en(9OyGT?$`l$Lk zjVM*9x7+X<@x?X2`JI{gaXfd9?HZkS^p1H8R!=&ABzE@+BrR~=bDph-bTN!5u=}_a%F@Tq}V4c2+SjEr<{%|`$ z&EH}x-9``#0Oj6CsBAlZyeM!ed>{81E;`&>D6*PCeVI9d zX#608Wog4qj>FDqlO!4Uhr2HRPCt)YtPHb?gPr{#Tw|ZHkDQ8F!-=WbOjPsgdNeX7 z)jRDaCj7aXbNbP3zd7o%msdI3Bm#{e8y# z5FGK;Sngkc^)>vOmKGmykjMY>%W#JPr6j71Mww`>l!JKBkF-+a zy3_lVAuU@P;cjX97;L&G>tpah6?h>rLr&0zD7lurrGnRU8SfmEfw<|oQJbs!oX2|f zE~q0!uqk&yc~!WEx;(c)*THeI`Uka@`6W}PBUx|8j2WptnIpFFoZ3kiF!+UgRFz7T z8r*`a?kteeS(vw!?}zlu#om0ZHLOZLwS4CDs-UDvoHsv3;Qdy;sdc&bb;G3PW~;s8 z6W}uy+wI$pTtdMlwP-qwoy~}UaC0hrEK)^`dezr9SFgJ8gxI97U~BC5EAzV%=x=0? zq0#swwG&s!&E?8VTf5^=m+43^IZa_?k#xe)(G-8yPbn;$Ial!42j|A7WAo&Qri*zY z?{E2=noLE+4oGX*&j&2>31yn&lSYOOe`16<9(-E=9cYp4omC_q?5RGS7 zO-JKtBkMm&O!>EDKK3Q5E1vHd{pSIWHyp?H)=BMVT~E+x^0UqsaO7AXSzdSI=~TF0 z`^!vLmW0{xH~8|-d3?#XoC>3p=OaiKIan$_6*RM*Ia}^^X!V2DHKQzSEFG4t+eRl$v9G^W%)^aJ;tq>YP2n@0a+S zANleJ>o;pT=_T;Z$ecR0Y;xO<91r}#C0UHM&HrA?TE)WN4bw}5WMS&Tj5T<)r+1T- z!t}cDo1Y-^3LE5k%Ub%m?1s%dSuc$3tg%aUCTC&p=%wi-tL|Uvh-IYb|NIFxU*dvIp zmT10{I9KxyUI0W9zOkL zGXJjq?e7{wSC%|5 zoX*xJZ#g>$?_ADdlTeXfb>6F*De@M5&r%<^$VlO^O#ak9F{_+i)B9fxpU`^x=0BiA zV^>SpW1hzO$EWtC10KCsYln5}-)ZCV!l%;f8_K>fpWg4Q=o-JPEt|7s$B(Ylrs9m> zU%o$2f68z46XQcKE`HqZ_UWg|aEo;_UycyQFf?*x`uwrJ+Nd|Zu#t7fS`eQW>W`fw%N2(?a`rN&Rw7s2!W%OXm%Pr4a4)!X9Y5+|?b z-r0~Xn|*)MTy$TWeiK}RIie@?EsfIRpM|VZV6B+t;lGW!ZH3YyiRmPr^^hN1Y^Qd^ z83x4=iL;`zbHCx4IiH_JH4 zMh4_g`M1e$E%%tdIo7`$|K29s{z$cL@tfMfa%|JbMNn0~>w7^&k;#)=zfM9;A#_Sl z1umPso~3i1@G={d$fI{3P1^`=ybZOy`3HmF|KOUje==oeJ98=gRM4x!=cibv^6aW& zuB*(0WfIH$Gu9vU&GKQqoxGfiYL>0*^ofAq@r}JSduYloPX0vpj|Zla9o85AIoX1$ z5!?||_WcXn^fKSx=GtGi=U-o5p`x7}Y%t{{v%Mcw2gfe^t~oZnZM$jPrk;_m)02MJ z@HaXot5mYj!K6bi-e>wL;nnlixNmyoSbtriJ8hA=je3SMl+E@RZv5Lp-GhT#oA1Lt z`QoxCcJW@cH<>dychj^7!w*b8+00MY!>5>ukF!Om&xh>vHamVRZAty9X~z!-)LY)N zZqxK$S^lD--QvG|^FcXKEsJ#)Vv|>K!-!LQ5wT_Rr$( zPg!g9St}uaYR%+k(3)Ko!X4!P;IeND89Q@s@W%ToqHrI`-H`^X-JqpZXOnRNk zhmgCOR6QebgN)2kU^dg>S-hbN2#Qr^iLJ)MWBMMmAB%uKiQB0ABfgW}c z$ks+k?I4gX-;g+)uDMN50|7I7PVHJl9856miL(n@L^qwjbw@)IrJzB+HGWR>z@XgQ z3(f14&F=h>Ih9#u7s-Os1Ie=(v`Wc7%|frM6KHh<$R>hPx2QaAdA)FY)qGcuzJ zbyg-|?D%n33+Mi@tt?F7+@F6^%N420gqxSle{kxgOu~%~%cuQmEl-Z9ZH-xyG3IIj zE4V}{*T&5*Nf<$Yj4*|W@NQ@8r%Ck1N{ z544+o2Hn%2A4#VQpsx7X4)7rfLPPGORI+s>YAw*S#+trE#@y4pYK016hQQW>9e|0ER#%(+ijJn&{X&(Xp@=vDd*63rPL;lx!c|9dlMHH zB(XMZc8+$3mi&qp+i*kIu1i&MTE}NRe_j)*sJxsNl;q!8=~%gV>C$zKM>-j)g#*|x zHlGjs))4Qta1e~NM}&z%Jm<;@@tg-Q_Hq_JbJIw=k|daR=Ih;Rz7@0oU*VdQ2}g@K z?`2zct#H4u&9FJ5ynnx4$C<0&A zn|lnU)~s1`D{c$xEF52stufTw;G_Qfvdvg}{bY4eLE&ofoo6*8J$1$T`IbOi>hkiK z*-1$@t;zvinhv_>FI;F5TqAKlSlQu7;?ZdHWg@jBW9KLM{u{(Zc#}hk+JWGZXD6QI znV}4f=q816z8;08|LyLdZ(!a1IS=3%d;JJkuy(CH zF4={*85JD!OnK#tp|ezxPkFL(=`omy=&_7N6z{@IzKx?>5Q+*^*QP- zmQ`hwyqL!BWP4xvwOo<6cYp9Mf2yUnaYNOi1v}T>TE==W~)tF3%1uc>|*3iE7vpapkYuB@at56c>S(InE}++voCZR)rp^e{L#1x3Q}$8V7lwj*iYS zgcJ{<51+f

`h+I`zFNr$BwJ4X-{;Q_yZyMnY7O zD_G4q_3HXtQi^gP>(C81pPcv@Xgd1+EBntszX8BaG6nByf+UUa0>q2}@;dcHXP?Iq z>RXDEKshuE&Fa%Nr?3p*xoa&fsyt7DZ?Cj}zdB%|-jmL(Gm-78xqcx@4=jMF%)j;YP z>@yV4vgK-~j#o%P^^~V(eIVQD{OKTR@&^9NDOmshCouWH?b}y!MN#})Vd)C%q~_$5 zib9mRyz{1@t8eRzapZ`~P6C6IzjroL z3}a&d`SYJ%@jtwudV|?#pRd}oMH~IqRXEq`skX)mCKXDoH=r&^Aq3i8t7CkCB|=J1 zE2eONXuer74|z2O z6T-{WNosd?%&fzht?C#7l@xRiFi{_CgD+s8H}5c@W4uP&IxEzuD$$!nmw{lOdeo7( zxXf-TKPI3==;Q6LHeZMHjrgF%ev9L)3{+l>05D?c5viZ81<+|msyEf~hEz&M zZ?Z&9CRJsfoSYInoPlVgm>bZ6^6FiH_8pzo@oSGIMxm`p$fC#8-7F+XOpu>r#k_q% zn~Ko2Ex}*yj@a$gz7Uy;&Nu82(7&{qr^9_5s=Ade$+gWFYjtWeT}Qm2VU&;Fe~%oo z)cUW|j=lHaV!#};7G=Qf&91{wBlOwt)i$?5RWapLYZMerh4Zd5YZ-g@N`$%&YTko5 zr*a;jsOXzB5CZf^oYV*u0Zn-m$uc9<*sDJ4kbhPhbM%|!(+{uF9 z1IF0f1Fs33xgiWS+Iz$x{bbFQ@Q=Qn)%2tAame&**!0wqDcaRE)# zH(xZ7Mq^(Cn$WA()bGFy3k&=De2&QCn>TL`r9|ex#2v)kEd1J(m&TKXMkCT+;&jCy zZ`{-pbV)gGnl)sI<5l(AwXy{JkLM-SKHG1Np-Sf%5SxVNynD#LI9)R`GQ1(mwD@q0 zme60?!qnpDU8Un(l#I@YxxBo*BTvu@mS>lDg;&1*XATbAd`(2`-p~fm=@-jSLEBl? zXCwt5T3Zd}Goz0s6(_o5V)9$`G~`rORkiiiq2*x}8qQmiJH~{iL82(MBK6z@IA)%i z;9iKLsm6Cvnr^deZ-KyWNV$IK>9LT{#CFU26nOjgZPQQIsM8DQkvi#a=u9sg_7_rn z87OUIkQjRd`%g~r*t7jXI8M}~$n9~5#LCe)NLocy_(ywJKDY}=D>li#-jR3W$w_Ce zoafJj-&)2=dWfEX+ufr68d87io}eKjoYi^nPhC--Q7v68dMTg?@`_g zg`EvY$xi0=xdW>D?2)z2LVHn|)2i1iMSl@<6LuI%H%sK*()lA!gmU3G(}X&c_Ve^#&FSBW}$JCXgtA1 z>`g4H;~uCjbxf6(AN}$tDtee_Dq0bvO-h}_BiMBM-R%c^14vJkjA_n#Hu#YC`iwOD z?%Ig#>}~WcZEbBwpPaZrV-(RWN@+W%!&a{Hz~?3~s>8@%5dcFx#YDQf4OJjDtbPJE z3j``;z0E44ZWg9#TBNirWF{i;g9X~}JlOe!m%8&%AKig^{u^vbQnY{+d|i3q-H#QA zbt|X`=g}b{RG*7ag-sgm6QIbnqb-Vbq~M&8q3?hJhm^`SlA1tWmU3SM&x;-JlCAEM zs4@HrJt$P2{ZX<*PnsQ0eH}UoNjV93-EHlHLa|o8wr-@x24*5Y>k;?{gh+q~LaMJJ zo2_<`uZ0YB3~o0AaDrpT<@`Hvd8z4xkfz_lOr-P^;z;mTs#DY(BnbG0@lwppjD)f2 zz}UyMHnDtNK0dxB;wKdoI~ECRJ;YMfVREo=l|$#dw=Eneanh_!Sm1?y*CSjZ5=~S@ z(E0uM-$^Oo6`IUo%J^ksMr&;kN1WZjMBcM!&mMtE0qbU0X9d;qaF*hRdl>Uj)R6-; z+ZY~)L`SFERi(2jXk6lKS6@=4<8E7oIRWRXCEDTau*-k z4>4h5xB_%|-9u?R8g!V=-gAHcSrsNt+q|lxlY_LUbLPpfXHc6c0?ioY44=_LiVOQ< zLUjc;)zSvzchufWm=G$zM5scYcf+10xKaX8(7xPmk%jgx>O{QO|E+-&ezX+Z-R?`{ z_PN~RyZSyAsUcvnaN=O+?lTcmU3@=0dvV01@5#&9ACb{+kFCMkA+Rs-xT5&V zl`DB3C4K9x)<5s*xzx`Kbdu8RViyjW_gx>@8YZf@HdV!?Oph~RFW+9Zpx z0}%&T2V^zGb&tj(fcb6dQd5ftaDxtDx9{}(iuvES#KRwx;#r}r29PiPUAuNAkJg@C zmetv9VAlfW`S^wuu0@Lsp+4lAcHr)=U+`NXAvN>|-aQV%X^*6&Br$Ze-9r%I(d^{m zF?o3I0^srBZyKzza7d2F-&q^)XZ56J4Nv@(%tZZtNrOwDx7DZD+@6CN56~2o)-BG@ zjSZ$;^-ElF)+USk;HIiqw+dJpvcy^7WOa{^toNueDYAbl)EpMYy#zg!H=D$8`qTId zJ|o(?8W?C~5e8z>hnAL8Ac8Qz?KXdzEr<3xDR|riIKRwli8uqJ?}#yM(lgLyoacKI z-NG-EEkB&gaK!}QLCqgEqL1D%{MzJ>wt%l-a4dxejzno`sTWqXEmcDDMQ(0&&!-8f z!gY4R7m1-?oO7q&+eEWtYS`RMY_WU=4-dyKm)Kl)9~Y_n>&lf)!n&EK zqqI`Q^CTnzkbL^S5y^K~l^YCl>6h-~eY`vU4gihwoKUfrm~~;^%(=-z2qW?b z0~A0`I{*k^r=OTZx7M|l>x0do-sY2XT$z+|?V89d9NXZo+!la;udp0OZ=YWmQ&4bQ zAnD5-Ed$_zSB<7_nWt5;x(&W3DTp$D4QJG~X#4BMYFwuRn6+H=qkYiQ=JxYKhX3x) z7E>j#GRCWoUqd7Mw4ykA0xezz-W7~I+YgG1gHg%;?K|N)a0tLb39ih5f$`wK2NH+W zmhi{%)u(qO+Vet>CpVl_)p{erhx*E)jlT@`G*~FQ%j-O^XbOWbc?45_1UEXPM3 zbMo@?u&IQ3AJDj8eE*%kPhI)qKw_PU^H=%!B)>i90kb*M^b-aYw?7@fuL}gE%RO|>4Ml6 zkBN*zxx>%TuQRt6N2gFd%FwqnyWe~Jhn)0lm_w=& z#p0i38y`I$SwVA>W%_s4L-Ep=AS&2Vv;wcFaZ4$fpHjq&VFEcmZKljB8 z!HpXew&iiD$Mo7JAYz2M-A9_(=#|Cb;g4~=j<4hqbU=^WkB2t|-{B^)0|fZO%+G}~ zZg@U5G-v;DMd%XA?j!zV#PC}2GhhxYiH3!#9Y#@wx*{Lif5(T0h3FdW25!G29EpdM7(pnN$1%Q&E zECQDxHFGKpQ4AVqwc#OYz;$qC+N)D!N)U3ye4lO=0eYB9h+XL|u(5v*s@0znuH=aWSGrcuJek0X1TR zE`mnc`>SE*`6Ccs3s)H?7!K^Le^Py#plgz2Lpt*affQ*#jF0xIFf%82ie_FUf}~j zaTcRyG0nX`)NT|FyLXW?Gp*o)iDv}!iMh;2ZtGvV+ljpspyR~A?y59Pstbp&T?aZJ zNLqHoEB)nKLU3SAkxLB2%#gY!g@wcLN#!W#fV5XzZ-Ly$?>_87%IMuhm!aOZ2kxE+ zt+Z1h9D%3cJ2e;3{@XuCxNfzO3EZPLI*pSz@&!OC7;=diLP_TmkfJ#^j2)PMQq zm+FOliU*OmR3L?*$u8z!4 zC2(2%W3j)sg@Nf!BKGua8P4A{5F=N>5jk5;RODwlQy~It_5jx65mtoKZJdTXlobrL z^!e2GZjxm?lZYLx=vSDwKE}MH-7|guufN^~Lo8y)1$K(lX(&5_Cn0h1gNTZ3lJ~A7 zr!myzl(*5e4y@l!*E!fKpaA-??-s9r>6U3rn+Z)rL2`5b>eV~AR>$|RO-?-- z-=0EF5d_E@!=F~FI2uAT!D+DCu&YobQk~d(6oNWBj>RTQGxJ3F_&#{}3>94wVkZjS zB3+lanVkT`;YPKovb8fbMgyLZLUg=u;zuio%0(cAI0m>x?a-m?V=)(kdHTTJMh$7N z)ODu&6`{`EKFgzVEXhq(bsd{DJ(lL0-ziilC03B^h%ueksh@CE9aB>9PWO{h0STT9 zND{iw+vq_sII7dtu_ReaL_~ycfYA60a~6>RO^BT1!HJ;~cjCNfAfp}3lrB1j*#S63 z2rXk^dC2tfw_#J1B>Lo@H~=)}5SaKxx~xgEsKR+1XuhV>j(s%TU#8OKYIDmSuLEJ# z1q2IkLsulRx2H->NS*PANJ$%XEL3qENZvQCb3PAG6ut7()p4Yx3eN%KCGf zh66uJT6puy>j18=gn1;K0xrb}(LCj?c(&y(U$%SdhX%=Atr@k-Xj&`HTXSy7k9#0a zMCOzW6nC(pO%az7pB_290PdfpWKE_X8$qkoV<{RM*TL%>mj_&`!t*zL9W4lMcRezUT0Z#pFeRAP@+&}G3L zbsZmSC88}-vPTpM)rZN|r30Ka1~foYe1`zKbPTrk{j~XGZJed7^I#OTfg@lM+c3-M z5Tc*pVT4m#=!GDnkpzR2;5RCzQ1;W-qxm%^^ybBHUL$mG!2Kw5r@2W48A9WHe57GQ z$qyq~s1cxgq*XHDVDW^z&Bj@u-laLvRIqU5;{iTzuaAcoHDS+?RuBM=+sJY{5G_&p zdNqTJ-%zRO9E%Nb$BuRWAUxzSZ>C+{&55OQd9QA)qiQecph|f8>OfZ2!l-(Tc}=g-%U$wD5=UmbSuuXHQe1DX9y?G)=%R2f^XbX^s(UQWR_^WI(N zeR%^{BAZ=K!z}S&O_cEg4o2}uqsj7&nYSX|cvp3mnSLWQG)6DSgLImfY;{Q{NgC!> zm(bS%*RXGZBvd>#R`cA1NDD`|TEwl_@f9zq&{g0%Z$!nHA`xJwhcE+$rte^f*Vp@$ zTuXs(z7IIl6c0DJgEp`UPLC7F1U$p`?d1h+j?e4s>z8bCdP+Jm442T}QmAP4;}Avy z6in?ambH=eD4E~*mrdbwGJKQ;ljJomE7~n=t9u|O6@Z0reWL*s)D`talbXb6I6~4f z8Ih@%=r)CDBxHOnjSw&fk~|0^e}p}1Fy8U8g~r0JE-sxuDzAzb2R22PxaFnl$O}~pcg5me6IK))fA!Afv_3G-Z#Z3-podq9u6puDEa#OAi ze+8A#He~Xuu-ICIX)up1Jv}{dG++X0jvT^362;p2WEI{z&?jrB`$llOk|>%YN^8&l z<42kV<+?1)y?G%eD}zDU7PVC%QQyDONEsiCH>q4{d{k)Tz$uKkQ|k9RfkR~mq1goNs^V;~2SvHEy^ zXT!NAYFuR6X#?};$!V7)Y*m4@skP&4Q1xs<}gWQ76%m)C9%L-oRM6(ro1ms81MPGrAf1QZ9sE_ z1QG*<2KR{!{+qo+BFH-v6}SYC%p8zzb@+26NKD^@ru`v?H3d>&^5*ulxfb_e&nf)l zAK&!FmJUgV(*Xb1q#3=lXsHoW`tn2KrMzVrK3dVQOuIN=UUNeF{Q2|mY6x=-LXC9g zA&@*!l%SD0${hi?_x^rcK>hE+w(F|u``W~8HEb5Lz=H`QKFAs$j$H2cU5|u5tCa;b za6Ey6xeT#mMa$MQ_PW>y}uC1$J2M$^heMn(?_sDg=SifOucz@yf^+-kHRY3eFu8&%ircLRsRN-F5&pI9~d(oUKh-lQ>aX7!T39^g3QDhr>{`=#{yf)*c@##U z-`v6U!eDAv>u8t*I9vvGhWkrL+*U<~jHb=Y5$%*(_*HvB@A(vgF%?C{$4M+m$^o&pzM^=za>y;S zFv8INyg0i3k(IT{ALlMbY71{}?gi50DWUMGoH4RU@ z=C)5f@@}<8Z*Kp=S8xU(myM(Wgv6250*&;5;y@C{<8dR`+BAuk2?2UuT2zLE?F9>V zGA$Y*1TZU#%Ji0@HiyibvUA}o2=Xr>M?UtgxFYU7u-Z1uQIvB|5E{|alYx`Bc=hVl z+@MhiR{?a@N1F%ML0r7FKGe_e1TdKtp80^M4#j9y`<_gPFdt_}Tq1?$tB~~7hu*qn zA_<~bi=n6#sy2xBY8Jz97KVYzfW-is&JBG&F?hh^ypx&bd@w%lA=3aZbI)h1^^qj# z-LFDH#P?*BPR9HDKU=NBQ8vJ7S?uQK#+mgk&ITxN6GDt?^Flxhz9-j1yjqXaN3HQN zdyKML#n1nuC4s#;7G^?L%QW>pHqxny^?MW)q@zqr?_Vs;>Otkg9HbmzC@6T;^)Kd% zLZXD#Lf3V}&jkBa$5w&dOr+1cVkVOR*-;*fx2r&+UPnp=R{0?XToYdpc27Eduy;u779 zW5BRw2<>stV2~8B?jZ|d=7-DRJiZDaNfbp7msuc1~i z5CT@?v_fyp1iiEGv)r@rwrPGTrabH&9+~h$FcXPKL6(X@^3i>2Xhw%53|$2xQP6dy zyHJaU^Rhe>4Noq#dbI)`s$AQl?g8wUD@vg6F=r8<{S=~ap4EF%et&5E_U;9mRtUF7 zGXyz1rV5+X7>SLI?%q-qb<1P4Wg*nrhJs8br1&XOhfB=w6t!ALAvE&(8@FyfqLG2{ z%XZbVa3pJWGt>EvVM?hDNpf=BsS0kh3PIdcrvVe{Ui8qT&m5HPh{FJpov z6wJF7pk$FsiZIuAfCE#ZZYZFam+kL z0U3c`?ujQpm(gbk2!1sKH>8d#B*oDHIY?>E1SEmVSlu4creo$WZ#Q;gq$R48hjAeB zi68nq``5!AsnIKkR5#wfA}9~3fCxy4jwaTgJoyl7glV2v0p{l3S7~sg)ulKnQR2)u4(6~&8Np&Kszi-`J2Wv%Qc0>Sk!i=X*z=3V|VDB&k z1;Yig=ak_Do$&67f*Mehfw;MA0ug*#f_01106vg<*3p`5+xbpx3>;hvLHXN|HkHGY zw?FnqsnrN(0vwErRX%d0L23$obQn7M`fAJyv*-|7h z|1qbeq@?BRU~jK@s7o&+V8Imv@iH=D_><+f9-gr6dS78&3*e&DVL*FK4R~HtiTs4^ z&`~PRroNS1Eg(?S^Rx8;2345pQ)GfVQA;sgOl;69#!FfoNqc0cRJ;xu_PB6Cx~@(A zjXrx2I1cb@yHO;I>%!*N)(21`tm}i{MI_th(4ICdq?3LoQKEd8f+TafYA2z1_q#FegtMf*%8aD8KOv?CZ?Ye!LP47=CUx8~ZU3fXSE#rOo{-sUmdpGd(V6CFxAfQ%Z_@N6qxVv#xflj3Ca4gm-+2PA zFnLsaL&2B9!A&#(D8P}ifB!~I^y#p!{qiA4yLi|aYy+*pJ2MpUzV`bF&luGfo z2VgEqe*ioms=s!pcEp(0Up)$fK~U`{W1yT=u*WovyD;Q8M09lsH%6gGII`kaS7Lsd zHy_ z=fDN_1)X6`(>hIsQ0qKVYUDpk=KA#MD5}qcnMT?QK#$ceEiDuERYK*A<=u32bRao! z5j<9>BH46u*ZPY+kZGJMfxPM?ywqr7M-hLg?JX;5jSm@OYReJ4XA%am zg&;zHNr{L`-inDW{cm=HZZ*1bV%94^*uMt1k^Nx#qZ;v$pOQ5a5C?$cT=az6nE`-M zm3(MqBoaZ+60nI7Rra3NZS2Gd`%(m^G1aXEU?DZz83dvT_SgV&o_1uWq+<zU-Q1c|#If5pNTHD}-iu%L^d8V7bCOMbpp0mCGP5k{| zF*yH2zU6;L5B>jgaUY_SA;Rmbj?e2<{W8z4y&Pf60iey*cUX^TxgW4)Dnd0HKOevI znYjS4qp@R%BmRWgO-n~dBsM8*j}BGU00>v}rF!El^;=8+CMc4qAq$jG0%cS*@`dz!5AEeE7BMJK6h@BJRM=QL2iY!=Epk0V%4&%aXn> zHK~KTt`1%@(!2+NlRKbiq7MHDj?oi2=OV;gn(O`L&@7WTh&NdXxDhZLml9>QirwYSsPIT*Lo6$ASGKd<6_k;2uRWqmq zLH)@Ig`wf$M+mu{P~JkG;NKO9)T9A|_pe!Jjy!O;1CpW`j4Y>+?~m?$pzBY zp83;H4{?RmuEZLe>=qCk?>6Q{qY$wz4nw(skZR;orU<^V!AQE8f2?N<(H0p8QIhNI z@$3w(b6jObuYH?dB8pN(CM#`dRiYfFIu?sM{3U1JY6cNosKVcYFk}TD7szlj&j>6d zWEVbTcHlMs4La#Z@N57y{F*Iwcz^p%k2k<>YalEz%}Ym$>kr)xkcw|3%X8FLhDc5L% zl#63cY7HbFAwMzXgO6AtFE6i-35AG)ZIbP;^5PiM;6O_D@K}#^EeNuPT6XX-h-*`J zfKqBAo1(h37Ib_Rlgccz#y<7h(HWq$3Aul~ZWyTCN$3M;#->8S!MOo`faXbFaBIX(DhtHnA)w9byWbf= zsI-Bf;u2K&(9!AEPD(~RW7l1)iTph-2?YX5n@SO!Q&G4B|7CUCbMkDaYB+(B9aEL7bAUlp6F{WnR1WxLi@4XWX&Y___l-2 z{uYbbX@-cT6jg&0^2I|R5XR7$xv{YTf_O+%WuwifeY6^|nYK%oE?DpY?&ZCnF7Z3+ zGn`G4UT{_&zy`BL_Y}=39ZaN@3!~`ml8bLFmHK@)+igpm9t#;ZHo+f&l->akg$hSN z#0dG=h)5jeU)Fd}fGNQ*!IC-%NuJ@M)-XFb5U^A`@cs^5D|>%a{_WNE*!NJkX}K@-A45aEYoKk6yl3w?^0ci+6L%!N{=IRsa$H2eh0r#&^pNq0i1^i z7b4t!&<#QITdX?G{z5Kdgscb5(o%%4wiv)^gtyCCV}@XuBS#g9DzL7|-IXKHD(d*9 zy;&_Km(E4?bwt@bGJS%s29QPWXI=hg6AAwb5FT_HUw+TN;mE8r{^R4nyHk}bXe}Y| zfYgE2Xi_r{O#jgpR<(ri9w71$7IOgQj5hGaWWbrHwYqTOf-%x564HrD@y1JtK)_CF zV5sykQ8TQ*%uSF4_8yE&JFNU+3h&j_?0TCSw2H9RRiuOiV<&C)4N$_h3KMk{?g^p* zxLRW<_#NaW{B(hwEazcoWZ9Sv@H<&E3hz*oL8?GvLQ|1%I%ba4m0L5tW
Eo-ys4z1uBC^D0k3o zg3-}Yf^zH7O-c29!s+nc&1SO5E3nVnK^X_+K_E0WbkjfKRN^KZ`5`K?6xI-513oGrXvT;1aS({;$iTr< z&p6E-M5IOcMV0?{RF={s#`2BHuli63yRVYEf&lABY%yK?3gMnibfkKuzgkNwKZXh) zM2bu_{KA{FYja_s_!&5ncG7k&SrcOx&hm(SE3;6}r0#Os6tq)~FTEfmhht65b?oKl zuC5J{6>IT7Pz4Y$(p)CLX6spAh}!Tfz7}YqPy%!2&rkhaDepQ=3!OBY0XO3p&3vj0 z3-{tCD6~T-AXrV>p9bOSEKO_%mcIwFp1%#_Pp`u}mjwd8t1WkHmSm|C3$n*8=MF4; z{yl1kN7QqVbB*n9|3p@&3cruA1GxUDA1=4-ruorRolC*yw!Zga=_BwT;@c{KjUkmN z!c-eFwGhVwcIzw`=y1sZ(?Ypw5!4UKZYo;4y57N_AeOqCvsKPaxu9JRLS>JwvMdg!{TfKc@ai%2^ z!7o|Sg~ImAMH!m8EmeLuZyHaO)>SNeG4u@l&-Az3h8q$*VdS_J*QZ!zbj&#%Ym>l> zAVM83jSvpo_+9={eRhYd7R7AUbx7)%*Ms}HvSRk zT-l5y-4^~BzPh^S2RIl%t+4w_rLcIt0{N!-sb&1a!o@Aah`CUI+z;*mcD?OUdGIZ0 zoC(eUCJ321zZbr7uOD*y_Ahd~56j=%<5OG{QurV$5P3pw$4ed_p5TtE0+~TptuerE2(_9)Ocu=|U0_^f!xr>_bx>c)kGBe?Y$eiKd z3l=Q6Au{4yWsdF_Mk&hU*U|805nF~r52KWjl`ES5h*MH|0vdW)5-x&X2(C>;lou06 zCT7r`xK{AvYZ-Llck($SF@l{T?i6#+VZ~rOcLO<0ZJn9YqUHF!(u2q^^1{q-hpQ&It;2R_{IGIpDFI+FMvry zaC3v7I&Pd$>7bB8JJUsz!d5iYvxu&sf;*<_N?{MP=Ce^Mf$YI#YK{~2GvZtA5T>o# zId9ATiSch)QUfxC<3d=KYF~#Y;7dTN?;tg1QOFBZMTIc*3#q|s62ysIu+B|680Dsic%W44mP^EgrD0QO(xUi? z@&zncs!Lr|Uoybkw{Dr8uV`VZctSIO5I*BiIoc&rRr-u*EU7{8YJ*iX>b-D6_xKqI zU#sv`vGgY3%JAO9`Z39MuOEnak53!QK0UIYrtl6RTrx$hXws0Gfq4A1t~~G=iuDmv z(uKI@&%Z~J80Om3+-+2#0WU-Z1_sL2CDkHQcniRkiaJ2~+c6XP4S2Lv*2AaLmPNTL z7_2)-wb=O!7ls4yBfEoY(_0emZ7W@kQsX{w%|dmYyEX1q#zqxwK($s zOw^x2oAC2OI^sM6#>ZU7?y#+{T33QZ_0I7XP+=nYC5ox_xKR(+w8InqU`m4o%UMHt ztJYw&>Ya09C?ldN1HE*fVkinYw9Di9m)1`pb_Xv3r&Q)?&$}l%KhK-zzU<1p`sG_;3KIrsy1(r(C9J#tT2 zEoo|M(wD=EfKKst53k+gKD5uIrgQqPmXs|2nVnsn{H}N?qTa=MKD;I0Hd~tje-bXa z4AB!WB3d|WEtNDn9xU%(ueZ;>b%GEfYVo}zC_XRa6!Yme!yHrKVTE=119i&D;+?APoFw3K?Fa+2TjUciu#G(EG zjwa}?j1g%Ql+_0igqr}Oqgb`@D_WUJBDEX^J;LVjY?q*u3aGyvxlMpRYDJMG28~;d zSLy3{1!pE{Ub`7beNfH*!8HAO9O(PKY zJOW>Xot-_k4SGZu!X`yDU{{KfnJB>Us>7HNu`@nrJ0F=+08(v17MV{}Y9_t5%cVsk zF;KGxFY4wpcJ@19ha9HY=Wscyy37N0UZQvfc3G+au;$E50?Ddr9cEN#sJG{ z8Ykh9v`cYtke`+V&?fbp_$dX=<;RgwV%+!m102ml^axGi$}DUFCwr5u9NRX&14 已超出稳定运行区间。 -5. **RPS=8 是稳定高吞吐边界**：total tok/s = 3875，E2E 延迟 4.54s，TTFT 仅 217ms，体验与吞吐兼顾。 -6. **EAGLE 投机采样稳定生效**：Accept Length 稳定在 2.60–2.62，有效降低了 decode 步数。 - -## 场景建议 - -| 场景 | 推荐配置 | 理由 | -|------|----------|------| -| 低延迟在线 API | `--max-concurrency 8` | TTFT 147ms，TPOT 8.4ms，E2E 1.79s | -| 平衡型在线服务 | `--max-concurrency 16` 或 `--request-rate 4` | 吞吐约 3k tok/s，E2E 2.6–1.7s | -| 高吞吐在线服务 | `--max-concurrency 32` 或 `--request-rate 8` | 吞吐 3.7–3.9k tok/s，E2E 4.3–4.5s | -| 最大吞吐/离线批处理 | `--max-concurrency 128` | 吞吐 7.1k tok/s，E2E 8.6s | -| 避免使用 | `--request-rate 16` | 系统过载，E2E 15s，TPOT 87ms | - -## 原始数据 - -- 汇总 JSON：`bench_results/sglang_8card_systematic_20260704_120819/summary.json` -- 每个测试点的详细结果与日志：`bench_results/sglang_8card_systematic_20260704_120819/` - -## 附录 A：Benchmark 参数说明 - -本次所有测试均使用 `sglang.bench_serving` 工具，参数统一如下： - -```bash -envs/sglang/bin/python -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 \ - --port 30000 \ - --dataset-name sharegpt \ - --dataset-path /data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json \ - --num-prompts 2000 \ - --model /data/models/DeepSeek-V4-Flash \ - --seed 42 -``` - -两个测试维度的区别仅在于最后一条参数： - -- **并发扫描**：`--max-concurrency {1,8,16,32,64,128}` - - 客户端同时保持 N 个未完成的请求，服务器队列按需堆积。 - - 适合模拟“系统能扛多少并发”的容量测试。 - -- **请求速率扫描**：`--request-rate {1,2,4,8,16}` - - 客户端按泊松分布以固定 RPS 发送请求。 - - 适合模拟真实线上按流量到达的场景。 - -### 指标定义 - -| 指标 | 含义 | -|------|------| -| **req/s** | 每秒完成的请求数 | -| **input tok/s** | 每秒处理的输入 token 数 | -| **output tok/s** | 每秒生成的输出 token 数 | -| **total tok/s** | input tok/s + output tok/s | -| **TTFT** | Time To First Token，首 token 延迟 | -| **TPOT** | Time Per Output Token，除首 token 外平均每个输出 token 的间隔 | -| **E2E Latency** | 端到端请求完成时间 | -| **Accept Length** | EAGLE 投机采样平均每次接受的 draft token 数 | - ---- - -## 附录 B：Token 成本计算 - -### 假设 - -- GPU 单价：**8.0 元 / 卡 / 小时** -- 本次部署使用 **8× NVIDIA H200** -- 整机每小时成本： - -``` -8 卡 × 8.0 元/卡/h = 64 元/h -``` - -### 公式 - -对于任意一个 benchmark 结果： - -``` -每小时处理 input tokens = input_tok/s × 3600 -每小时生成 output tokens = output_tok/s × 3600 - -每 1M input tokens 成本 = 64 / (input_tok/s × 3600 / 1,000,000) -每 1M output tokens 成本 = 64 / (output_tok/s × 3600 / 1,000,000) -``` - -### 各场景成本 - -| 场景 | input tok/s | output tok/s | 1M input cost | 1M output cost | -|------|-------------|--------------|---------------|----------------| -| c=1 | 335.83 | 243.51 | 52.94 元 | 73.01 元 | -| c=8 | 1,283.33 | 930.54 | 13.85 元 | 19.10 元 | -| c=16 | 1,736.31 | 1,259.00 | 10.24 元 | 14.12 元 | -| c=32 | 2,151.87 | 1,560.32 | 8.26 元 | 11.39 元 | -| c=64 | 2,775.39 | 2,012.43 | 6.41 元 | 8.83 元 | -| c=128 | 4,132.36 | 2,996.37 | 4.30 元 | 5.93 元 | -| rps=1 | 286.41 | 207.67 | 62.07 元 | 85.61 元 | -| rps=2 | 571.97 | 414.74 | 31.08 元 | 42.86 元 | -| rps=4 | 1,139.69 | 826.39 | 15.60 元 | 21.51 元 | -| rps=8 | 2,246.34 | 1,628.82 | 7.91 元 | 10.91 元 | -| rps=16 | 4,176.94 | 3,028.70 | 4.26 元 | 5.87 元 | - -### 对外定价建议 - -裸 GPU 成本不等于对外售价。建议根据 SLA 和毛利率加价： - -| 推荐场景 | 裸成本（input/output） | 说明 | -|----------|------------------------|------| -| 低延迟在线（c=8） | 13.85 元 / 19.10 元 per 1M | TTFT 147ms，TPOT 8.4ms，适合实时聊天 | -| 平衡型在线（c=16） | 10.24 元 / 14.12 元 per 1M | 吞吐与延迟兼顾，主流 API 可参考此档位 | -| 高吞吐在线（c=32 / rps=8） | 8.09 元 / 11.15 元 per 1M | 吞吐高、延迟可接受，适合批量在线服务 | -| 最大吞吐/离线（c=128） | 4.30 元 / 5.93 元 per 1M | 延迟 8.6s，仅适合离线批处理 | - -> 注意：`rps=16` 虽已接近最大吞吐，但系统已明显过载（E2E 15s），不建议按此成本对外定价。 - ---- - -## 附录 C：最大吞吐压测（优化服务参数） - -为了探索成本下限，重新部署了服务并调优了参数： - -```bash -envs/sglang/bin/sglang serve \ - --trust-remote-code \ - --model-path /data/models/DeepSeek-V4-Flash \ - --tp 8 \ - --moe-runner-backend marlin \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --max-running-requests 512 \ - --chunked-prefill-size 16384 \ - --host 0.0.0.0 \ - --port 30000 -``` - -与附录 A/B 的测试相比，主要变化： -- `--max-running-requests` 从默认 256 提升到 **512** -- `--chunked-prefill-size` 从默认 8192 提升到 **16384** - -### 超大并发扫描结果 - -| 并发 | 耗时(s) | req/s | input tok/s | output tok/s | total tok/s | Mean TTFT | Mean TPOT | Mean E2E(ms) | -|------|---------|-------|-------------|--------------|-------------|-----------|-----------|--------------| -| 128 | 155.26 | 12.88 | 3,710.50 | 2,690.48 | 6,400.98 | 410.55 ms | 48.45 ms | 9,696.87 | -| 256 | 90.65 | 22.06 | 6,355.48 | 4,608.35 | 10,963.84 | 509.73 ms | 62.71 ms | 10,869.76 | -| 512 | 71.39 | 28.02 | 8,070.04 | 5,851.58 | 13,921.61 | 1,343.73 ms| 121.20 ms | 16,176.14 | -| **1024** | **68.21** | **29.32** | **8,445.53** | **6,123.84** | **14,569.37** | **13,265.46 ms** | **105.36 ms** | **27,566.99** | - -### 关键发现 - -1. **高并发下吞吐显著提升**：优化参数后，c=256 及以上远超原配置的最高值（7,129 tok/s）；c=1024 达到 **14,569 tok/s**，约为原配置峰值的两倍。 -2. **c=1024 达到最高吞吐**：total **14,569 tok/s**，output **6,124 tok/s**，但 TTFT 13.3s，系统已严重过载。 -3. **c=512 是实际可用上限**：total **13,922 tok/s**，output **5,852 tok/s**，虽然 TTFT 1.3s、E2E 16s，但吞吐接近峰值，适合离线批处理。 -4. **c=256 是高压在线的甜点**：total **10,964 tok/s**，TTFT 510ms，E2E 10.9s，如果业务能容忍 10 秒级响应，这是性价比很高的点。 - ---- - -## 附录 D：最低 Token 成本（基于最大吞吐） - -仍按 **8.0 元/卡时、8 卡整机 64 元/小时** 计算。 - -### 公式 - -``` -每 1M input tokens 成本 = 64 / (input_tok/s × 3600 / 1,000,000) -每 1M output tokens 成本 = 64 / (output_tok/s × 3600 / 1,000,000) -``` - -### 最大吞吐场景成本 - -| 场景 | input tok/s | output tok/s | 1M input cost | 1M output cost | -|------|-------------|--------------|---------------|----------------| -| c=128 | 3,710.50 | 2,690.48 | 4.79 元 | 6.61 元 | -| c=256 | 6,355.48 | 4,608.35 | 2.80 元 | 3.86 元 | -| c=512 | 8,070.04 | 5,851.58 | 2.20 元 | 3.04 元 | -| **c=1024** | **8,445.53** | **6,123.84** | **2.10 元** | **2.90 元** | - -### 结论 - -在当前硬件和优化参数下： - -- **理论最低 input 成本**：约 **2.10 元 / 1M input tokens**（c=1024） -- **理论最低 output 成本**：约 **2.90 元 / 1M output tokens**（c=1024） -- **实际可用最低成本**：c=512 时 **2.20 元 / 1M input**、**3.04 元 / 1M output**，此时吞吐已达峰值 95% 以上，且延迟比 c=1024 可控得多。 - -> 注意：c=1024 的 TTFT 超过 13 秒，仅适合完全不在意延迟的离线批处理任务；对外 API 服务不建议按此成本定价。 - ---- - -## 附录 E：测试数据长度分布 - -本次所有 benchmark 使用的是 `ShareGPT_V4.3_unfiltered_cleaned_split.json` 数据集。`sglang.bench_serving` 在 `--num-prompts 2000` 且 `--seed 42` 的条件下，从数据集中采样出 **2000 条真实对话请求**作为测试负载。 - -### 长度统计 - -| 指标 | Input Tokens | Output Tokens | -|------|--------------|---------------| -| 请求数 | 2,000 | 2,000 | -| 平均值 | 288.05 | 208.86 | -| 标准差 | 420.07 | 217.60 | -| 最小值 | 2 | 2 | -| P50 | 125 | 150 | -| P90 | 723 | 516 | -| P95 | 841 | 667 | -| P99 | 2,190 | 811 | -| 最大值 | 3,996 | 1,655 | -| **总量** | **576,098** | **417,728** | - -### 分布特点 - -1. **输入长度呈现典型长尾分布**：P50 仅 125 tokens，但 P99 达到 2,190 tokens，最大值 3,996 tokens。说明 ShareGPT 中既有大量短 prompt，也有少量长上下文对话。 -2. **输出长度相对集中**：P50 150 tokens，P90 516 tokens，P99 811 tokens，最大值 1,655 tokens。大部分回复属于中等长度。 -3. **输出/输入比约为 0.73**：平均每条请求输出 209 tokens、输入 288 tokens，整体负载偏 decode 侧。 -4. **与原始 ShareGPT 全集的差异**：原始全集输出长度均值约 1,122 tokens，而 bench 采样后的输出均值仅 209 tokens。这是因为 `sglang.bench_serving` 在处理 sharegpt 数据集时会根据内置规则对输出做截断/采样，以更贴近典型在线 serving 负载。 - -### 长度分布图 - -![长度分布](length_distribution.png) - -> 图表文件：`bench_results/sglang_8card_systematic_20260704_120819/length_distribution.png` diff --git a/experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/summary.json b/experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/summary.json deleted file mode 100644 index af4117a..0000000 --- a/experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/summary.json +++ /dev/null @@ -1,156 +0,0 @@ -[ - { - "name": "sharegpt_concurrency_1", - "args": "--max-concurrency 1", - "duration_s": 1715.45, - "request_throughput": 1.17, - "input_token_throughput": 335.83, - "output_token_throughput": 243.51, - "total_token_throughput": 579.34, - "mean_ttft_ms": 128.09, - "mean_tpot_ms": 4.0, - "mean_e2e_latency_ms": 856.77, - "accept_length": 2.62, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_1.json" - }, - { - "name": "sharegpt_concurrency_8", - "args": "--max-concurrency 8", - "duration_s": 448.91, - "request_throughput": 4.46, - "input_token_throughput": 1283.33, - "output_token_throughput": 930.54, - "total_token_throughput": 2213.87, - "mean_ttft_ms": 146.76, - "mean_tpot_ms": 8.41, - "mean_e2e_latency_ms": 1790.65, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_8.json" - }, - { - "name": "sharegpt_concurrency_16", - "args": "--max-concurrency 16", - "duration_s": 331.79, - "request_throughput": 6.03, - "input_token_throughput": 1736.31, - "output_token_throughput": 1259.0, - "total_token_throughput": 2995.3, - "mean_ttft_ms": 168.95, - "mean_tpot_ms": 12.67, - "mean_e2e_latency_ms": 2642.1, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_16.json" - }, - { - "name": "sharegpt_concurrency_32", - "args": "--max-concurrency 32", - "duration_s": 267.72, - "request_throughput": 7.47, - "input_token_throughput": 2151.87, - "output_token_throughput": 1560.32, - "total_token_throughput": 3712.19, - "mean_ttft_ms": 206.24, - "mean_tpot_ms": 20.72, - "mean_e2e_latency_ms": 4249.71, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_32.json" - }, - { - "name": "sharegpt_concurrency_64", - "args": "--max-concurrency 64", - "duration_s": 207.57, - "request_throughput": 9.64, - "input_token_throughput": 2775.39, - "output_token_throughput": 2012.43, - "total_token_throughput": 4787.82, - "mean_ttft_ms": 248.41, - "mean_tpot_ms": 32.61, - "mean_e2e_latency_ms": 6557.39, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_64.json" - }, - { - "name": "sharegpt_concurrency_128", - "args": "--max-concurrency 128", - "duration_s": 139.41, - "request_throughput": 14.35, - "input_token_throughput": 4132.36, - "output_token_throughput": 2996.37, - "total_token_throughput": 7128.73, - "mean_ttft_ms": 330.4, - "mean_tpot_ms": 46.15, - "mean_e2e_latency_ms": 8596.12, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_128.json" - }, - { - "name": "sharegpt_rps_1", - "args": "--request-rate 1", - "duration_s": 2011.46, - "request_throughput": 0.99, - "input_token_throughput": 286.41, - "output_token_throughput": 207.67, - "total_token_throughput": 494.08, - "mean_ttft_ms": 146.87, - "mean_tpot_ms": 4.91, - "mean_e2e_latency_ms": 1007.62, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_1.json" - }, - { - "name": "sharegpt_rps_2", - "args": "--request-rate 2", - "duration_s": 1007.21, - "request_throughput": 1.99, - "input_token_throughput": 571.97, - "output_token_throughput": 414.74, - "total_token_throughput": 986.71, - "mean_ttft_ms": 160.63, - "mean_tpot_ms": 5.99, - "mean_e2e_latency_ms": 1205.24, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_2.json" - }, - { - "name": "sharegpt_rps_4", - "args": "--request-rate 4", - "duration_s": 505.49, - "request_throughput": 3.96, - "input_token_throughput": 1139.69, - "output_token_throughput": 826.39, - "total_token_throughput": 1966.08, - "mean_ttft_ms": 174.67, - "mean_tpot_ms": 9.02, - "mean_e2e_latency_ms": 1743.4, - "accept_length": 2.6, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_4.json" - }, - { - "name": "sharegpt_rps_8", - "args": "--request-rate 8", - "duration_s": 256.46, - "request_throughput": 7.8, - "input_token_throughput": 2246.34, - "output_token_throughput": 1628.82, - "total_token_throughput": 3875.15, - "mean_ttft_ms": 216.65, - "mean_tpot_ms": 25.35, - "mean_e2e_latency_ms": 4542.64, - "accept_length": 2.6, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_8.json" - }, - { - "name": "sharegpt_rps_16", - "args": "--request-rate 16", - "duration_s": 137.92, - "request_throughput": 14.5, - "input_token_throughput": 4176.94, - "output_token_throughput": 3028.7, - "total_token_throughput": 7205.64, - "mean_ttft_ms": 377.28, - "mean_tpot_ms": 87.16, - "mean_e2e_latency_ms": 15048.09, - "accept_length": 2.61, - "output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_16.json" - } -] \ No newline at end of file diff --git a/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121046/summary.json b/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121046/summary.json deleted file mode 100644 index 20dfa8b..0000000 --- a/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121046/summary.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "model": "/data/models/Qwen3-4B", - "draft_model": "deepseek-ai/dspark_qwen3_4b_block7", - "url": "http://127.0.0.1:30003/v1/completions", - "max_tokens": 64, - "num_prompts": 20, - "dataset": "/data/user1/yy/datasets/ShareGPT_filtered_chat.json", - "timestamp": "20260705_121046", - "results": [ - { - "concurrency": 1, - "num_prompts": 20, - "duration_s": 0.07108968612737954, - "success_count": 0, - "error": "all requests failed", - "errors": [ - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'" - ] - }, - { - "concurrency": 4, - "num_prompts": 20, - "duration_s": 0.036050053080543876, - "success_count": 0, - "error": "all requests failed", - "errors": [ - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'", - "404, message='Not Found', url='http://127.0.0.1:30003/v1/completions'" - ] - } - ] -} \ No newline at end of file diff --git a/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121218/summary.json b/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121218/summary.json deleted file mode 100644 index c719bcc..0000000 --- a/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121218/summary.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "model": "/data/models/Qwen3-4B", - "draft_model": "deepseek-ai/dspark_qwen3_4b_block7", - "url": "http://127.0.0.1:30003/v1/completions", - "max_tokens": 64, - "num_prompts": 20, - "dataset": "/data/user1/yy/datasets/ShareGPT_filtered_chat.json", - "timestamp": "20260705_121218", - "results": [ - { - "concurrency": 1, - "num_prompts": 20, - "duration_s": 3.1493232590146363, - "success_count": 20, - "fail_count": 0, - "input_tokens": 6132, - "output_tokens": 456, - "total_tokens": 6588, - "request_throughput": 6.350570695704836, - "input_throughput": 1947.0849753031027, - "output_throughput": 144.79301186207027, - "total_throughput": 2091.877987165173, - "mean_ttft_ms": 55.51949484506622, - "p50_ttft_ms": 27.64087135437876, - "p99_ttft_ms": 295.08487760089326, - "mean_tpot_ms": 4.6249661131817925, - "p50_tpot_ms": 4.634099831345484, - "p99_tpot_ms": 4.827384204164935, - "mean_e2e_ms": 156.8067875574343, - "p50_e2e_ms": 148.02213886287063, - "p99_e2e_ms": 406.6163514624348, - "errors": [] - }, - { - "concurrency": 4, - "num_prompts": 20, - "duration_s": 1.0271000929642469, - "success_count": 20, - "fail_count": 0, - "input_tokens": 6132, - "output_tokens": 450, - "total_tokens": 6582, - "request_throughput": 19.472298889857267, - "input_throughput": 5970.206839630238, - "output_throughput": 438.1267250217885, - "total_throughput": 6408.333564652026, - "mean_ttft_ms": 82.42244796128944, - "p50_ttft_ms": 37.77908312622458, - "p99_ttft_ms": 263.8731326162815, - "mean_tpot_ms": 5.172366561559164, - "p50_tpot_ms": 5.150655882704692, - "p99_tpot_ms": 5.708390201389117, - "mean_e2e_ms": 193.68963547749445, - "p50_e2e_ms": 153.2427944475785, - "p99_e2e_ms": 387.3923780536279, - "errors": [] - } - ] -} \ No newline at end of file diff --git a/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121256/summary.json b/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121256/summary.json deleted file mode 100644 index 61edc71..0000000 --- a/experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121256/summary.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "model": "/data/models/Qwen3-4B", - "draft_model": "deepseek-ai/dspark_qwen3_4b_block7", - "url": "http://127.0.0.1:30003/v1/completions", - "max_tokens": 256, - "num_prompts": 500, - "dataset": "/data/user1/yy/datasets/ShareGPT_filtered_chat.json", - "timestamp": "20260705_121256", - "results": [ - { - "concurrency": 1, - "num_prompts": 500, - "duration_s": 181.22913801996037, - "success_count": 500, - "fail_count": 0, - "input_tokens": 108545, - "output_tokens": 36681, - "total_tokens": 145226, - "request_throughput": 2.7589382450460618, - "input_throughput": 598.9379036170495, - "output_throughput": 202.4012275330692, - "total_throughput": 801.3391311501188, - "mean_ttft_ms": 17.586736707482487, - "p50_ttft_ms": 15.236841514706612, - "p99_ttft_ms": 43.418583339080215, - "mean_tpot_ms": 4.756160026498775, - "p50_tpot_ms": 4.747943201144331, - "p99_tpot_ms": 4.845208972490266, - "mean_e2e_ms": 361.93272305326536, - "p50_e2e_ms": 369.4158981088549, - "p99_e2e_ms": 552.2739849891514, - "errors": [] - }, - { - "concurrency": 4, - "num_prompts": 500, - "duration_s": 49.20666282507591, - "success_count": 500, - "fail_count": 0, - "input_tokens": 108545, - "output_tokens": 36991, - "total_tokens": 145536, - "request_throughput": 10.161225559584139, - "input_throughput": 2205.900456730121, - "output_throughput": 751.7477893491538, - "total_throughput": 2957.6482460792745, - "mean_ttft_ms": 26.531575095374137, - "p50_ttft_ms": 24.292025598697364, - "p99_ttft_ms": 41.29285377450287, - "mean_tpot_ms": 5.007692315964597, - "p50_tpot_ms": 4.998277087638343, - "p99_tpot_ms": 5.130429720718591, - "mean_e2e_ms": 392.14430282171816, - "p50_e2e_ms": 400.28460952453315, - "p99_e2e_ms": 625.0063395127652, - "errors": [] - }, - { - "concurrency": 16, - "num_prompts": 500, - "duration_s": 14.20007478701882, - "success_count": 500, - "fail_count": 0, - "input_tokens": 108545, - "output_tokens": 36529, - "total_tokens": 145074, - "request_throughput": 35.21108215972787, - "input_throughput": 7643.973826055324, - "output_throughput": 2572.451240425399, - "total_throughput": 10216.425066480724, - "mean_ttft_ms": 31.879239567089826, - "p50_ttft_ms": 28.826430439949036, - "p99_ttft_ms": 74.12134333979331, - "mean_tpot_ms": 5.771187238080647, - "p50_tpot_ms": 5.76495784573639, - "p99_tpot_ms": 6.1421910060182805, - "mean_e2e_ms": 447.76513252267614, - "p50_e2e_ms": 459.7155440133065, - "p99_e2e_ms": 669.6232496039011, - "errors": [] - }, - { - "concurrency": 64, - "num_prompts": 500, - "duration_s": 7.960247536888346, - "success_count": 500, - "fail_count": 0, - "input_tokens": 108545, - "output_tokens": 36583, - "total_tokens": 145128, - "request_throughput": 62.81211704572815, - "input_throughput": 13635.882489457124, - "output_throughput": 4595.711355767746, - "total_throughput": 18231.59384522487, - "mean_ttft_ms": 73.04830395104364, - "p50_ttft_ms": 53.465207340195775, - "p99_ttft_ms": 252.97995713772252, - "mean_tpot_ms": 12.42267764661875, - "p50_tpot_ms": 12.61188557741512, - "p99_tpot_ms": 13.063075978620708, - "mean_e2e_ms": 968.1061067078263, - "p50_e2e_ms": 967.5668340642005, - "p99_e2e_ms": 1512.9108975501729, - "errors": [] - }, - { - "concurrency": 128, - "num_prompts": 500, - "duration_s": 7.319521516095847, - "success_count": 500, - "fail_count": 0, - "input_tokens": 108545, - "output_tokens": 36613, - "total_tokens": 145158, - "request_throughput": 68.31047615619204, - "input_throughput": 14829.52126874773, - "output_throughput": 5002.102927013319, - "total_throughput": 19831.62419576105, - "mean_ttft_ms": 145.49234842788428, - "p50_ttft_ms": 90.75760398991406, - "p99_ttft_ms": 423.2492828951217, - "mean_tpot_ms": 22.034148180580335, - "p50_tpot_ms": 23.052740126916806, - "p99_tpot_ms": 23.587740010073134, - "mean_e2e_ms": 1730.9650285840034, - "p50_e2e_ms": 1733.5386699996889, - "p99_e2e_ms": 2722.885905068833, - "errors": [] - } - ] -} \ No newline at end of file diff --git a/scripts/benchmark_dspark_0707/README.md b/scripts/benchmark_dspark_0707/README.md deleted file mode 100644 index 08cbf61..0000000 --- a/scripts/benchmark_dspark_0707/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# DSpark Benchmark Scripts (2026-07-07) - -## Location - -`/data/user1/yy/scripts/benchmark_dspark_0707` - -## Scripts - -| Script | Purpose | Output Location | -|---|---|---| -| `bench_dspark_p1.sh` | Quick benchmark: chat_standard, generation_standard, summarization | `bench_results/dspark_grid_${RUN_ID}/p1_quick/...` | -| `bench_dspark_p2.sh` | Core benchmark: chat_short, rag_medium, long_context_probe | `bench_results/dspark_grid_${RUN_ID}/p2_core/...` | -| `bench_dspark_p3.sh` | Extension benchmark: stress_standard, decode_heavy | `bench_results/dspark_grid_${RUN_ID}/p3_extension/...` | -| `bench_dspark_focused.sh` | Focused sweep used for spec-tokens comparison | `bench_results/dspark_grid_${RUN_ID}/focused/...` or `bench_results/dspark_st_comparison_${RUN_ID}/focused/...` | -| `run_dspark_benchmark_grid.sh` | Orchestrator: starts vllm-dspark server, runs P1/P2/P3, stops server | `bench_results/dspark_grid_${RUN_ID}/` | -| `run_dspark_st_comparison.sh` | Orchestrator: compares `--spec-tokens 3` vs `5` | `bench_results/dspark_st_comparison_${RUN_ID}/` | -| `parse_results.py` | Parses grid logs and generates `report.md` | Writes to `bench_results//report.md` | -| `parse_st_comparison.py` | Parses spec-tokens comparison logs | Writes `comparison_report.md` under result root | -| `parse_eagle_vs_dspark.py` | Merges DSpark st comparison with SGLang EAGLE results | Writes `dspark_vs_eagle_report.md` under EAGLE result root | - -## Common Environment - -- Python env: `/data/user1/yy/envs/vllm-dspark` for server -- Benchmark client: `/data/user1/yy/envs/sglang/bin/python -m sglang.bench_serving --backend vllm` -- Model: `/data/models/DeepSeek-V4-Flash-DSpark` -- Default port: `30004` -- Default warmup: `100` requests - -## Usage - -### Full grid benchmark - -```bash -bash scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh -``` - -Results land in `bench_results/dspark_grid_YYYYMMDD-HHMMSS/`. - -### Spec-tokens comparison - -```bash -bash scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh -``` - -Results land in `bench_results/dspark_st_comparison_YYYYMMDD-HHMMSS/`. - -### Parse existing results - -```bash -# Grid -/data/user1/yy/envs/sglang/bin/python scripts/benchmark_dspark_0707/parse_results.py /data/user1/yy/bench_results/dspark_grid_ - -# Spec-tokens comparison -/data/user1/yy/envs/sglang/bin/python scripts/benchmark_dspark_0707/parse_st_comparison.py /data/user1/yy/bench_results/dspark_st_comparison_ -``` - -## Notes - -- These scripts were originally located in `/data/user1/yy/benchmark_dspark_0707/` and moved here on 2026-07-08. -- `run_dspark_benchmark_grid.sh` and `run_dspark_st_comparison.sh` assume server start scripts are in `scripts/` (one directory up). diff --git a/scripts/benchmark_dspark_0707/bench_dspark_focused.sh b/scripts/benchmark_dspark_0707/bench_dspark_focused.sh deleted file mode 100755 index 0c4a124..0000000 --- a/scripts/benchmark_dspark_0707/bench_dspark_focused.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# Focused benchmark for spec-tokens comparison. -# Covers short/medium/long input and low/medium/high concurrency. - -BACKEND="vllm" -PORT="${PORT:-30004}" -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -BENCH_PY="/data/user1/yy/envs/sglang/bin/python" -DATASET="/data/user1/yy/datasets/ShareGPT_filtered_chat.json" -RESULT_ROOT="${RESULT_ROOT:-/data/user1/yy/bench_results/dspark_grid}" -WARMUP_REQUESTS="${WARMUP_REQUESTS:-100}" -PHASE="focused" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" - -num_prompts() { - local c="$1" - if (( c <= 4 )); then - local n=$((c * 8)) - (( n < 32 )) && n=32 - echo "${n}" - elif (( c <= 32 )); then - echo $((c * 16)) - else - echo $((c * 8)) - fi -} - -run_case() { - local scenario="$1" - local input_len="$2" - local output_len="$3" - local concurrencies="$4" - for concurrency in ${concurrencies}; do - local prompts - prompts="$(num_prompts "${concurrency}")" - local out_dir="${RESULT_ROOT}/${PHASE}/${scenario}/${RUN_ID}" - mkdir -p "${out_dir}" - echo "phase=${PHASE} scenario=${scenario} input=${input_len} output=${output_len} concurrency=${concurrency} prompts=${prompts} warmup=${WARMUP_REQUESTS}" - "${BENCH_PY}" -m sglang.bench_serving \ - --backend "${BACKEND}" \ - --host 127.0.0.1 \ - --port "${PORT}" \ - --model "${MODEL}" \ - --dataset-name random \ - --dataset-path "${DATASET}" \ - --random-input-len "${input_len}" \ - --random-output-len "${output_len}" \ - --num-prompts "${prompts}" \ - --max-concurrency "${concurrency}" \ - --warmup-requests "${WARMUP_REQUESTS}" \ - 2>&1 | tee "${out_dir}/c${concurrency}.log" - done -} - -curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null -run_case "chat_short" 512 256 "1 8 16 32 64" -run_case "chat_standard" 1000 256 "64" -run_case "generation_standard" 1000 1000 "64" -run_case "rag_medium" 4000 512 "1 8 32" -run_case "long_context_probe" 16000 512 "1 4 8" -run_case "stress_standard" 1000 256 "64 96 128" -run_case "decode_heavy" 512 2000 "1 32" diff --git a/scripts/benchmark_dspark_0707/bench_dspark_p1.sh b/scripts/benchmark_dspark_0707/bench_dspark_p1.sh deleted file mode 100755 index 45505f6..0000000 --- a/scripts/benchmark_dspark_0707/bench_dspark_p1.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# P1 quick benchmark for vllm-dspark -# Mirrored from benchmark_grid_0707/h200_vllm_p1_grid.sh - -BACKEND="vllm" -PORT="${PORT:-30004}" -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -BENCH_PY="/data/user1/yy/envs/sglang/bin/python" -DATASET="/data/user1/yy/datasets/ShareGPT_filtered_chat.json" -RESULT_ROOT="${RESULT_ROOT:-/data/user1/yy/bench_results/dspark_grid}" -WARMUP_REQUESTS="${WARMUP_REQUESTS:-100}" -PHASE="p1_quick" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" - -num_prompts() { - local c="$1" - if (( c <= 4 )); then - local n=$((c * 8)) - (( n < 32 )) && n=32 - echo "${n}" - elif (( c <= 32 )); then - echo $((c * 16)) - else - echo $((c * 8)) - fi -} - -run_case() { - local scenario="$1" - local input_len="$2" - local output_len="$3" - local concurrencies="$4" - for concurrency in ${concurrencies}; do - local prompts - prompts="$(num_prompts "${concurrency}")" - local out_dir="${RESULT_ROOT}/${PHASE}/${scenario}/${RUN_ID}" - mkdir -p "${out_dir}" - echo "phase=${PHASE} scenario=${scenario} input=${input_len} output=${output_len} concurrency=${concurrency} prompts=${prompts}" - "${BENCH_PY}" -m sglang.bench_serving \ - --backend "${BACKEND}" \ - --host 127.0.0.1 \ - --port "${PORT}" \ - --model "${MODEL}" \ - --dataset-name random \ - --dataset-path "${DATASET}" \ - --random-input-len "${input_len}" \ - --random-output-len "${output_len}" \ - --num-prompts "${prompts}" \ - --max-concurrency "${concurrency}" \ - --warmup-requests "${WARMUP_REQUESTS}" \ - 2>&1 | tee "${out_dir}/c${concurrency}.log" - done -} - -curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null -run_case "chat_standard" 1000 256 "1 8 16 32" -run_case "generation_standard" 1000 1000 "1 8 16 32" -run_case "summarization" 8000 1000 "1 4 8 16" diff --git a/scripts/benchmark_dspark_0707/bench_dspark_p2.sh b/scripts/benchmark_dspark_0707/bench_dspark_p2.sh deleted file mode 100755 index 5a2be94..0000000 --- a/scripts/benchmark_dspark_0707/bench_dspark_p2.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# P2 core benchmark for vllm-dspark -# Mirrored from benchmark_grid_0707/h200_vllm_p2_grid.sh - -BACKEND="vllm" -PORT="${PORT:-30004}" -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -BENCH_PY="/data/user1/yy/envs/sglang/bin/python" -DATASET="/data/user1/yy/datasets/ShareGPT_filtered_chat.json" -RESULT_ROOT="${RESULT_ROOT:-/data/user1/yy/bench_results/dspark_grid}" -WARMUP_REQUESTS="${WARMUP_REQUESTS:-100}" -PHASE="p2_core" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" - -num_prompts() { - local c="$1" - if (( c <= 4 )); then - local n=$((c * 8)) - (( n < 32 )) && n=32 - echo "${n}" - elif (( c <= 32 )); then - echo $((c * 16)) - else - echo $((c * 8)) - fi -} - -run_case() { - local scenario="$1" - local input_len="$2" - local output_len="$3" - local concurrencies="$4" - for concurrency in ${concurrencies}; do - local prompts - prompts="$(num_prompts "${concurrency}")" - local out_dir="${RESULT_ROOT}/${PHASE}/${scenario}/${RUN_ID}" - mkdir -p "${out_dir}" - echo "phase=${PHASE} scenario=${scenario} input=${input_len} output=${output_len} concurrency=${concurrency} prompts=${prompts}" - "${BENCH_PY}" -m sglang.bench_serving \ - --backend "${BACKEND}" \ - --host 127.0.0.1 \ - --port "${PORT}" \ - --model "${MODEL}" \ - --dataset-name random \ - --dataset-path "${DATASET}" \ - --random-input-len "${input_len}" \ - --random-output-len "${output_len}" \ - --num-prompts "${prompts}" \ - --max-concurrency "${concurrency}" \ - --warmup-requests "${WARMUP_REQUESTS}" \ - 2>&1 | tee "${out_dir}/c${concurrency}.log" - done -} - -curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null -run_case "chat_short" 512 256 "1 8 16 32 64" -run_case "chat_standard" 1000 256 "64" -run_case "generation_standard" 1000 1000 "64" -run_case "rag_medium" 4000 512 "1 4 8 16 32" -run_case "summarization" 8000 1000 "32" -run_case "long_context_probe" 16000 512 "1 2 4 8 16" diff --git a/scripts/benchmark_dspark_0707/bench_dspark_p3.sh b/scripts/benchmark_dspark_0707/bench_dspark_p3.sh deleted file mode 100755 index b4df410..0000000 --- a/scripts/benchmark_dspark_0707/bench_dspark_p3.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# P3 extension benchmark for vllm-dspark -# Mirrored from benchmark_grid_0707/h200_vllm_p3_grid.sh - -BACKEND="vllm" -PORT="${PORT:-30004}" -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -BENCH_PY="/data/user1/yy/envs/sglang/bin/python" -DATASET="/data/user1/yy/datasets/ShareGPT_filtered_chat.json" -RESULT_ROOT="${RESULT_ROOT:-/data/user1/yy/bench_results/dspark_grid}" -WARMUP_REQUESTS="${WARMUP_REQUESTS:-100}" -PHASE="p3_extension" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" - -num_prompts() { - local c="$1" - if (( c <= 4 )); then - local n=$((c * 8)) - (( n < 32 )) && n=32 - echo "${n}" - elif (( c <= 32 )); then - echo $((c * 16)) - else - echo $((c * 8)) - fi -} - -run_case() { - local scenario="$1" - local input_len="$2" - local output_len="$3" - local concurrencies="$4" - for concurrency in ${concurrencies}; do - local prompts - prompts="$(num_prompts "${concurrency}")" - local out_dir="${RESULT_ROOT}/${PHASE}/${scenario}/${RUN_ID}" - mkdir -p "${out_dir}" - echo "phase=${PHASE} scenario=${scenario} input=${input_len} output=${output_len} concurrency=${concurrency} prompts=${prompts}" - "${BENCH_PY}" -m sglang.bench_serving \ - --backend "${BACKEND}" \ - --host 127.0.0.1 \ - --port "${PORT}" \ - --model "${MODEL}" \ - --dataset-name random \ - --dataset-path "${DATASET}" \ - --random-input-len "${input_len}" \ - --random-output-len "${output_len}" \ - --num-prompts "${prompts}" \ - --max-concurrency "${concurrency}" \ - --warmup-requests "${WARMUP_REQUESTS}" \ - 2>&1 | tee "${out_dir}/c${concurrency}.log" - done -} - -curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null -run_case "decode_heavy" 512 2000 "1 8 16 32" -run_case "long_rag" 32000 512 "1 2 4 8" -run_case "stress_standard" 1000 256 "96 128" -run_case "stress_generation" 1000 1000 "96 128" diff --git a/scripts/benchmark_dspark_0707/parse_eagle_vs_dspark.py b/scripts/benchmark_dspark_0707/parse_eagle_vs_dspark.py deleted file mode 100644 index 8bf10dc..0000000 --- a/scripts/benchmark_dspark_0707/parse_eagle_vs_dspark.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python3 -"""Parse EAGLE focused benchmark logs and merge with DSpark st=3/st=5 report.""" - -import re -from pathlib import Path - -EAGLE_ROOT = Path("/data/user1/yy/bench_results/eagle_grid/focused") -DSPARK_REPORT = Path("/data/user1/yy/bench_results/dspark_st_comparison_20260707-150649/comparison_report.md") -OUT_REPORT = Path("/data/user1/yy/bench_results/eagle_grid/dspark_vs_eagle_report.md") - - -def parse_log(log_path: Path) -> dict: - text = log_path.read_text(errors="ignore") - lines = text.splitlines() - header_idx = None - for i, line in enumerate(lines): - if "============ Serving Benchmark Result ==========" in line: - header_idx = i - break - if header_idx is None: - return {} - section = "\n".join(lines[header_idx:]) - - def get_float(pattern): - m = re.search(pattern + r"\s+([\d.]+)", section) - return float(m.group(1)) if m else None - - def get_int(pattern): - m = re.search(pattern + r"\s+(\d+)", section) - return int(m.group(1)) if m else None - - return { - "duration_s": get_float(r"Benchmark duration \(s\):"), - "successful_requests": get_int(r"Successful requests:"), - "req_throughput": get_float(r"Request throughput \(req/s\):"), - "input_tok_throughput": get_float(r"Input token throughput \(tok/s\):"), - "output_tok_throughput": get_float(r"Output token throughput \(tok/s\):"), - "total_tok_throughput": get_float(r"Total token throughput \(tok/s\):"), - "mean_e2e_ms": get_float(r"Mean E2E Latency \(ms\):"), - "p95_e2e_ms": get_float(r"P95 E2E Latency \(ms\):"), - "p99_e2e_ms": get_float(r"P99 E2E Latency \(ms\):"), - "mean_ttft_ms": get_float(r"Mean TTFT \(ms\):"), - "p95_ttft_ms": get_float(r"P95 TTFT \(ms\):"), - "p99_ttft_ms": get_float(r"P99 TTFT \(ms\):"), - "mean_tpot_ms": get_float(r"Mean TPOT \(ms\):"), - "p95_tpot_ms": get_float(r"P95 TPOT \(ms\):"), - "p99_tpot_ms": get_float(r"P99 TPOT \(ms\):"), - "mean_itl_ms": get_float(r"Mean ITL \(ms\):"), - } - - -def parse_dspark_report(path: Path) -> list: - """Extract rows from the DSpark comparison markdown table.""" - rows = [] - text = path.read_text(errors="ignore") - in_table = False - for line in text.splitlines(): - if "| Scenario | Concurrency | Spec" in line: - in_table = True - continue - if in_table and line.startswith("|"): - parts = [p.strip() for p in line.split("|") if p.strip()] - if len(parts) < 14 or parts[0] == "Scenario" or set(parts[0]) <= set("-:"): - continue - rows.append({ - "scenario": parts[0], - "concurrency": int(parts[1]), - "spec": parts[2], - "duration_s": float(parts[3]), - "req_throughput": float(parts[4]), - "output_tok_throughput": float(parts[5]), - "total_tok_throughput": float(parts[6]), - "mean_e2e_ms": float(parts[7]), - "p95_e2e_ms": float(parts[8]), - "p99_e2e_ms": float(parts[9]), - "mean_ttft_ms": float(parts[10]), - "p99_ttft_ms": float(parts[11]), - "mean_tpot_ms": float(parts[12]), - "p99_tpot_ms": float(parts[13]), - }) - return rows - - -def parse_eagle_focused() -> list: - rows = [] - if not EAGLE_ROOT.exists(): - return rows - for scenario_dir in sorted(EAGLE_ROOT.iterdir()): - if not scenario_dir.is_dir(): - continue - scenario = scenario_dir.name - # Use latest run_id only - run_ids = sorted([p.name for p in scenario_dir.iterdir() if p.is_dir()]) - if not run_ids: - continue - run_id = run_ids[-1] - run_dir = scenario_dir / run_id - for log in sorted(run_dir.glob("c*.log"), key=lambda p: int(p.stem[1:])): - concurrency = int(log.stem[1:]) - m = parse_log(log) - if not m: - continue - rows.append({ - "scenario": scenario, - "concurrency": concurrency, - "spec": "EAGLE", - **m, - }) - return rows - - -def pick_best_dspark(dspark_rows: list, scenario: str, concurrency: int) -> dict: - candidates = [ - r for r in dspark_rows - if r["scenario"] == scenario and r["concurrency"] == concurrency - ] - if not candidates: - return None - # Pick the spec-tokens config with highest total tok/s - return max(candidates, key=lambda r: r["total_tok_throughput"]) - - -def fmt(v, digits=2): - if v is None: - return "N/A" - return f"{v:.{digits}f}" - - -def main(): - dspark_rows = parse_dspark_report(DSPARK_REPORT) - eagle_rows = parse_eagle_focused() - - # Determine comparison pairs - pairs = [] - for er in eagle_rows: - dr = pick_best_dspark(dspark_rows, er["scenario"], er["concurrency"]) - if dr: - pairs.append((er, dr)) - - lines = [ - "# DSpark vs SGLang EAGLE 投机解码对比报告", - "", - "- DSpark 结果：`/data/user1/yy/bench_results/dspark_st_comparison_20260707-150649`", - "- EAGLE 结果：`/data/user1/yy/bench_results/eagle_grid/focused`", - "- DSpark 模型：`/data/models/DeepSeek-V4-Flash-DSpark`", - "- EAGLE 模型：`/data/models/DeepSeek-V4-Flash`", - "- DSpark 后端：vllm-dspark (TP=8, FP8 KV cache, spec-method=dspark)", - "- EAGLE 后端：SGLang (TP=8, FP8 KV cache, speculative-algorithm=EAGLE, num_steps=3, topk=1, draft_tokens=4)", - "- 压测客户端：`sglang.bench_serving`", - "- Warmup：100 条", - "", - "## 核心指标对比", - "", - "| Scenario | Concurrency | Method | Total tok/s | Out tok/s | Req/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", - ] - - for er, dr in pairs: - lines.append( - f"| {er['scenario']} | {er['concurrency']} | DSpark(st={dr['spec']}) | " - f"{fmt(dr['total_tok_throughput'])} | {fmt(dr['output_tok_throughput'])} | {fmt(dr['req_throughput'])} | " - f"{fmt(dr['mean_e2e_ms'])} | {fmt(dr['p95_e2e_ms'])} | {fmt(dr['p99_e2e_ms'])} | " - f"{fmt(dr['mean_ttft_ms'])} | {fmt(dr['p99_ttft_ms'])} | {fmt(dr['mean_tpot_ms'])} | {fmt(dr['p99_tpot_ms'])} |" - ) - lines.append( - f"| {er['scenario']} | {er['concurrency']} | EAGLE | " - f"{fmt(er['total_tok_throughput'])} | {fmt(er['output_tok_throughput'])} | {fmt(er['req_throughput'])} | " - f"{fmt(er['mean_e2e_ms'])} | {fmt(er['p95_e2e_ms'])} | {fmt(er['p99_e2e_ms'])} | " - f"{fmt(er['mean_ttft_ms'])} | {fmt(er['p99_ttft_ms'])} | {fmt(er['mean_tpot_ms'])} | {fmt(er['p99_tpot_ms'])} |" - ) - - # Winner analysis - winner_lines = [] - for er, dr in pairs: - eagle_total = er["total_tok_throughput"] or 0 - dspark_total = dr["total_tok_throughput"] or 0 - if eagle_total > dspark_total: - winner = "EAGLE" - delta = (eagle_total - dspark_total) / dspark_total * 100 - else: - winner = f"DSpark(st={dr['spec']})" - delta = (dspark_total - eagle_total) / eagle_total * 100 - winner_lines.append({ - "scenario": er["scenario"], - "concurrency": er["concurrency"], - "winner": winner, - "dspark_total": dspark_total, - "eagle_total": eagle_total, - "delta": delta, - }) - - lines += [ - "", - "## 吞吐 Winner 统计（按 Total tok/s）", - "", - "| Scenario | Concurrency | Winner | DSpark Total tok/s | EAGLE Total tok/s | 提升 |", - "|---|---:|---:|---:|---:|---:|", - ] - for w in winner_lines: - lines.append( - f"| {w['scenario']} | {w['concurrency']} | {w['winner']} | " - f"{fmt(w['dspark_total'])} | {fmt(w['eagle_total'])} | {fmt(w['delta'])}% |" - ) - - # Summary counts - eagle_wins = sum(1 for w in winner_lines if w["winner"].startswith("EAGLE")) - dspark_wins = len(winner_lines) - eagle_wins - - lines += [ - "", - "## 关键发现", - "", - f"- 对比点数：{len(pairs)}", - f"- EAGLE 胜出：{eagle_wins} 个", - f"- DSpark 胜出：{dspark_wins} 个", - "", - ] - - if eagle_wins > dspark_wins: - lines.append("**结论：在相同负载下，SGLang EAGLE 的综合吞吐优于 vllm-dspark。**") - elif dspark_wins > eagle_wins: - lines.append("**结论：在相同负载下，vllm-dspark 的综合吞吐优于 SGLang EAGLE。**") - else: - lines.append("**结论：两者综合吞吐相当，各有优劣。**") - - lines += [ - "", - "### 延迟与稳定性观察", - "", - ] - - # Compute average ratios - avg_e2e_ratio = sum((er["mean_e2e_ms"] or 0) / (dr["mean_e2e_ms"] or 1) for er, dr in pairs) / len(pairs) - avg_tpot_ratio = sum((er["mean_tpot_ms"] or 0) / (dr["mean_tpot_ms"] or 1) for er, dr in pairs) / len(pairs) - avg_ttft_ratio = sum((er["mean_ttft_ms"] or 0) / (dr["mean_ttft_ms"] or 1) for er, dr in pairs) / len(pairs) - - lines += [ - f"- 平均 Mean E2E 比值（EAGLE / DSpark）：{fmt(avg_e2e_ratio)}", - f"- 平均 Mean TPOT 比值（EAGLE / DSpark）：{fmt(avg_tpot_ratio)}", - f"- 平均 Mean TTFT 比值（EAGLE / DSpark）：{fmt(avg_ttft_ratio)}", - "", - "> 比值 < 1 表示 EAGLE 更快；> 1 表示 DSpark 更快。", - "", - "## 优化建议", - "", - "1. 如果以吞吐为首要目标，当前配置下 **vllm-dspark 更优**，建议继续使用 `--spec-tokens` 并根据并发选择 3（高并发）或 5（低并发/长上下文）。", - "2. SGLang EAGLE 本次表现不佳，平均延迟和 TPOT 均显著高于 DSpark；如需进一步评估 EAGLE，可尝试调整 `--speculative-num-steps`、`--speculative-eagle-topk`、`--speculative-num-draft-tokens` 或更换 MOE runner backend，并确认是否已完成 `sglang.compile_deep_gemm` 预编译。", - "3. 注意两套后端的实现差异（CUDA graph、KV cache 管理、调度器、draft 模型架构）会显著影响不同并发和输入长度下的表现，建议按实际业务负载做最终选型。", - "", - ] - - OUT_REPORT.write_text("\n".join(lines)) - print("\n".join(lines)) - print(f"\nReport saved to: {OUT_REPORT}", file=__import__("sys").stderr) - - -if __name__ == "__main__": - main() diff --git a/scripts/benchmark_dspark_0707/parse_results.py b/scripts/benchmark_dspark_0707/parse_results.py deleted file mode 100644 index d85fefa..0000000 --- a/scripts/benchmark_dspark_0707/parse_results.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -"""Parse dspark benchmark grid logs and generate a markdown report.""" - -import os -import re -import sys -from pathlib import Path - - -def parse_log(log_path: Path) -> dict: - text = log_path.read_text(errors="ignore") - lines = text.splitlines() - - # Find benchmark header line - header_idx = None - for i, line in enumerate(lines): - if "============ Serving Benchmark Result ============" in line: - header_idx = i - break - if header_idx is None: - return {} - - section = "\n".join(lines[header_idx:]) - - def get_float(pattern): - m = re.search(pattern + r"\s+([\d.]+)", section) - return float(m.group(1)) if m else None - - def get_int(pattern): - m = re.search(pattern + r"\s+(\d+)", section) - return int(m.group(1)) if m else None - - return { - "duration_s": get_float(r"Benchmark duration \(s\):"), - "successful_requests": get_int(r"Successful requests:"), - "req_throughput": get_float(r"Request throughput \(req/s\):"), - "input_tok_throughput": get_float(r"Input token throughput \(tok/s\):"), - "output_tok_throughput": get_float(r"Output token throughput \(tok/s\):"), - "total_tok_throughput": get_float(r"Total token throughput \(tok/s\):"), - "mean_e2e_ms": get_float(r"Mean E2E Latency \(ms\):"), - "p95_e2e_ms": get_float(r"P95 E2E Latency \(ms\):"), - "p99_e2e_ms": get_float(r"P99 E2E Latency \(ms\):"), - "mean_ttft_ms": get_float(r"Mean TTFT \(ms\):"), - "p95_ttft_ms": get_float(r"P95 TTFT \(ms\):"), - "p99_ttft_ms": get_float(r"P99 TTFT \(ms\):"), - "mean_tpot_ms": get_float(r"Mean TPOT \(ms\):"), - "p95_tpot_ms": get_float(r"P95 TPOT \(ms\):"), - "p99_tpot_ms": get_float(r"P99 TPOT \(ms\):"), - "mean_itl_ms": get_float(r"Mean ITL \(ms\):"), - } - - -def main(): - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/user1/yy/experiments/legacy_bench_results/dspark_grid_20260707-132641") - if not result_root.exists(): - print(f"Result root not found: {result_root}", file=sys.stderr) - sys.exit(1) - - phases = ["p1_quick", "p2_core", "p3_extension"] - report_lines = [ - "# vllm-dspark Benchmark Grid Report", - "", - f"- Result root: `{result_root}`", - f"- Model: `/data/models/DeepSeek-V4-Flash-DSpark`", - f"- Backend: vllm-dspark (TP=8, FP8 KV cache, spec-method=dspark, spec-tokens=5)", - f"- Benchmark client: `sglang.bench_serving --backend vllm`", - "", - ] - - for phase in phases: - phase_dir = result_root / phase - if not phase_dir.exists(): - continue - - report_lines.append(f"## {phase}") - report_lines.append("") - - scenarios = sorted([p.name for p in phase_dir.iterdir() if p.is_dir()]) - for scenario in scenarios: - scenario_dir = phase_dir / scenario - run_ids = sorted([p.name for p in scenario_dir.iterdir() if p.is_dir()]) - for run_id in run_ids: - run_dir = scenario_dir / run_id - logs = sorted(run_dir.glob("c*.log"), key=lambda p: int(p.stem[1:])) - - report_lines.append(f"### {scenario} ({run_id})") - report_lines.append("") - report_lines.append("| Concurrency | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean ITL(ms) |") - report_lines.append("|------------|------------|---------|------|---------|----------|------------|-------------|-------------|-------------|--------------|-------------|-------------|--------------|-------------|-------------|-------------|") - - for log in logs: - concurrency = int(log.stem[1:]) - metrics = parse_log(log) - if not metrics: - continue - - def fmt(v): - return f"{v:.2f}" if v is not None else "N/A" - - report_lines.append( - f"| {concurrency} | {fmt(metrics['duration_s'])} | {metrics['successful_requests'] or 'N/A'} | " - f"{fmt(metrics['req_throughput'])} | {fmt(metrics['input_tok_throughput'])} | " - f"{fmt(metrics['output_tok_throughput'])} | {fmt(metrics['total_tok_throughput'])} | " - f"{fmt(metrics['mean_e2e_ms'])} | {fmt(metrics['p95_e2e_ms'])} | {fmt(metrics['p99_e2e_ms'])} | " - f"{fmt(metrics['mean_ttft_ms'])} | {fmt(metrics['p95_ttft_ms'])} | {fmt(metrics['p99_ttft_ms'])} | " - f"{fmt(metrics['mean_tpot_ms'])} | {fmt(metrics['p95_tpot_ms'])} | {fmt(metrics['p99_tpot_ms'])} | " - f"{fmt(metrics['mean_itl_ms'])} |" - ) - - report_lines.append("") - - report_text = "\n".join(report_lines) - report_path = result_root / "report.md" - report_path.write_text(report_text) - print(report_text) - print(f"\nReport saved to: {report_path}", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/scripts/benchmark_dspark_0707/parse_st_comparison.py b/scripts/benchmark_dspark_0707/parse_st_comparison.py deleted file mode 100644 index d8aecaa..0000000 --- a/scripts/benchmark_dspark_0707/parse_st_comparison.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Parse spec-tokens 3 vs 5 comparison results and generate a report.""" - -import os -import re -import sys -from pathlib import Path - - -def parse_log(log_path: Path) -> dict: - text = log_path.read_text(errors="ignore") - lines = text.splitlines() - - header_idx = None - for i, line in enumerate(lines): - if "============ Serving Benchmark Result ============" in line: - header_idx = i - break - if header_idx is None: - return {} - - section = "\n".join(lines[header_idx:]) - - def get_float(pattern): - m = re.search(pattern + r"\s+([\d.]+)", section) - return float(m.group(1)) if m else None - - def get_int(pattern): - m = re.search(pattern + r"\s+(\d+)", section) - return int(m.group(1)) if m else None - - return { - "duration_s": get_float(r"Benchmark duration \(s\):"), - "successful_requests": get_int(r"Successful requests:"), - "req_throughput": get_float(r"Request throughput \(req/s\):"), - "input_tok_throughput": get_float(r"Input token throughput \(tok/s\):"), - "output_tok_throughput": get_float(r"Output token throughput \(tok/s\):"), - "total_tok_throughput": get_float(r"Total token throughput \(tok/s\):"), - "mean_e2e_ms": get_float(r"Mean E2E Latency \(ms\):"), - "p95_e2e_ms": get_float(r"P95 E2E Latency \(ms\):"), - "p99_e2e_ms": get_float(r"P99 E2E Latency \(ms\):"), - "mean_ttft_ms": get_float(r"Mean TTFT \(ms\):"), - "p95_ttft_ms": get_float(r"P95 TTFT \(ms\):"), - "p99_ttft_ms": get_float(r"P99 TTFT \(ms\):"), - "mean_tpot_ms": get_float(r"Mean TPOT \(ms\):"), - "p95_tpot_ms": get_float(r"P95 TPOT \(ms\):"), - "p99_tpot_ms": get_float(r"P99 TPOT \(ms\):"), - "mean_itl_ms": get_float(r"Mean ITL \(ms\):"), - } - - -def collect_results(result_root: Path) -> dict: - """Collect results into {(scenario, concurrency, spec_tokens): metrics}.""" - results = {} - focused_dir = result_root / "focused" - if not focused_dir.exists(): - return results - - for scenario_dir in focused_dir.iterdir(): - if not scenario_dir.is_dir(): - continue - scenario = scenario_dir.name - for run_dir in scenario_dir.iterdir(): - if not run_dir.is_dir(): - continue - run_id = run_dir.name - # run_id format: _st{3,5} - if "_st" not in run_id: - continue - spec_tokens = run_id.split("_st")[-1] - for log in run_dir.glob("c*.log"): - concurrency = int(log.stem[1:]) - metrics = parse_log(log) - if metrics: - results[(scenario, concurrency, spec_tokens)] = metrics - return results - - -def fmt(v): - return f"{v:.2f}" if v is not None else "N/A" - - -def main(): - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/user1/yy/bench_results/dspark_st_comparison_20260707-150649") - if not result_root.exists(): - print(f"Result root not found: {result_root}", file=sys.stderr) - sys.exit(1) - - results = collect_results(result_root) - if not results: - print("No results found.", file=sys.stderr) - sys.exit(1) - - # Group by scenario and concurrency - keys = sorted({(s, c) for (s, c, _) in results.keys()}) - - report_lines = [ - "# vllm-dspark `--spec-tokens` 对比报告", - "", - f"- 结果目录：`{result_root}`", - f"- 模型：`/data/models/DeepSeek-V4-Flash-DSpark`", - f"- 后端：vllm-dspark (TP=8, FP8 KV cache)", - f"- 对比参数：`--spec-tokens 3` vs `--spec-tokens 5`", - f"- Warmup：100 条", - f"- 压测客户端：`sglang.bench_serving --backend vllm`", - "", - "## 核心指标对比", - "", - "| Scenario | Concurrency | Spec | Duration(s) | Req/s | Out tok/s | Total tok/s | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Mean TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", - ] - - for scenario, concurrency in keys: - for st in ["3", "5"]: - m = results.get((scenario, concurrency, st), {}) - if not m: - continue - report_lines.append( - f"| {scenario} | {concurrency} | {st} | {fmt(m['duration_s'])} | {fmt(m['req_throughput'])} | " - f"{fmt(m['output_tok_throughput'])} | {fmt(m['total_tok_throughput'])} | {fmt(m['mean_e2e_ms'])} | " - f"{fmt(m['p95_e2e_ms'])} | {fmt(m['p99_e2e_ms'])} | {fmt(m['mean_ttft_ms'])} | {fmt(m['p99_ttft_ms'])} | " - f"{fmt(m['mean_tpot_ms'])} | {fmt(m['p99_tpot_ms'])} |" - ) - - # Summary table: best total tok/s per scenario/concurrency - report_lines.append("") - report_lines.append("## 吞吐 winner 统计") - report_lines.append("") - report_lines.append("| Scenario | Concurrency | Winner (Total tok/s) | st=3 Total tok/s | st=5 Total tok/s | 提升 |") - report_lines.append("|---|---:|---:|---:|---:|---:|") - - for scenario, concurrency in keys: - m3 = results.get((scenario, concurrency, "3"), {}) - m5 = results.get((scenario, concurrency, "5"), {}) - t3 = m3.get("total_tok_throughput", 0) or 0 - t5 = m5.get("total_tok_throughput", 0) or 0 - if t3 == 0 and t5 == 0: - continue - winner = "st=3" if t3 > t5 else "st=5" - uplift = (max(t3, t5) / min(t3, t5) - 1) * 100 if min(t3, t5) > 0 else 0 - report_lines.append( - f"| {scenario} | {concurrency} | {winner} | {fmt(t3)} | {fmt(t5)} | {fmt(uplift)}% |" - ) - - report_lines.append("") - report_lines.append("## 观察与建议") - report_lines.append("") - - # Compute overall wins - st3_wins = 0 - st5_wins = 0 - for scenario, concurrency in keys: - m3 = results.get((scenario, concurrency, "3"), {}) - m5 = results.get((scenario, concurrency, "5"), {}) - t3 = m3.get("total_tok_throughput", 0) or 0 - t5 = m5.get("total_tok_throughput", 0) or 0 - if t3 > t5: - st3_wins += 1 - elif t5 > t3: - st5_wins += 1 - - report_lines.append(f"- **st=3 在 {st3_wins} 个配置中吞吐更高，st=5 在 {st5_wins} 个配置中吞吐更高。**") - report_lines.append("- 具体结论请根据上表分析。") - report_lines.append("") - - report_text = "\n".join(report_lines) - report_path = result_root / "comparison_report.md" - report_path.write_text(report_text) - print(report_text) - print(f"\nComparison report saved to: {report_path}", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh b/scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh deleted file mode 100755 index 4170d3c..0000000 --- a/scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# Run H200 DeepSeek-V4-Flash-DSpark benchmark grid in the current environment. -# This orchestrator starts the vllm-dspark server, runs P1/P2/P3 benchmarks, -# and stops the server afterwards. -# -# Environment: -# - vllm-dspark env: /data/user1/yy/envs/vllm-dspark -# - benchmark client: /data/user1/yy/envs/sglang/bin/python -m sglang.bench_serving -# - model: /data/models/DeepSeek-V4-Flash-DSpark -# - server port: 30004 -# -# Usage: -# bash run_dspark_benchmark_grid.sh -# -# To only run benchmarks against an already-running server: -# SKIP_MANAGE_SERVER=1 bash run_dspark_benchmark_grid.sh - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -START_SCRIPT="${SCRIPT_DIR}/../start_dsv4_dspark_8card.sh" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-/data/user1/yy/bench_results/dspark_grid_${RUN_ID}}" -LOG_DIR="${RESULT_ROOT}/logs" -PORT="${PORT:-30004}" -PID_FILE="/data/user1/yy/dsv4_flash_dspark.pid" - -mkdir -p "${LOG_DIR}" - -log() { - echo "[$(date --iso-8601=seconds)] $*" | tee -a "${LOG_DIR}/orchestrator.log" -} - -is_server_healthy() { - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 -} - -stop_server() { - if [[ -f "${PID_FILE}" ]]; then - local pid - pid="$(cat "${PID_FILE}")" - if kill -0 "${pid}" 2>/dev/null; then - log "stopping existing dspark server pid=${pid}" - kill "${pid}" 2>/dev/null || true - sleep 5 - kill -9 "${pid}" 2>/dev/null || true - fi - rm -f "${PID_FILE}" - fi - # Fallback: kill any lingering vllm serve processes for this model - pkill -9 -f "vllm serve.*DeepSeek-V4-Flash-DSpark" 2>/dev/null || true - sleep 2 -} - -start_server() { - log "starting dspark server with ${START_SCRIPT}" - if [[ ! -x "${START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${START_SCRIPT}" - exit 1 - fi - bash "${START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - if ! is_server_healthy; then - log "error: dspark server failed to become healthy" - exit 1 - fi - log "dspark server is healthy" -} - -run_phase() { - local name="$1" - local script="$2" - local log_file="${LOG_DIR}/${name}.log" - - log "running ${name}: ${script}" - RUN_ID="${RUN_ID}" RESULT_ROOT="${RESULT_ROOT}" bash "${script}" 2>&1 | tee "${log_file}" - log "finished ${name}" -} - -on_exit() { - local code="$?" - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_server - fi - exit "${code}" -} -trap on_exit EXIT - -main() { - log "run_id=${RUN_ID}" - log "result_root=${RESULT_ROOT}" - log "script_dir=${SCRIPT_DIR}" - - if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming server is already running" - if ! is_server_healthy; then - log "error: no healthy server found at port ${PORT}" - exit 1 - fi - else - stop_server - start_server - fi - - log "===== DSpark BENCHMARK START =====" - run_phase "dspark_p1" "${SCRIPT_DIR}/bench_dspark_p1.sh" - run_phase "dspark_p2" "${SCRIPT_DIR}/bench_dspark_p2.sh" - run_phase "dspark_p3" "${SCRIPT_DIR}/bench_dspark_p3.sh" - log "===== DSpark BENCHMARK DONE =====" - - log "all results saved to ${RESULT_ROOT}" -} - -main "$@" diff --git a/scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh b/scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh deleted file mode 100755 index 747c166..0000000 --- a/scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# Compare vllm-dspark with --spec-tokens 3 vs 5. -# Uses the parameterized start script and focused benchmark. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -START_SCRIPT="${SCRIPT_DIR}/../start_dsv4_dspark_8card_param.sh" -BENCH_SCRIPT="${SCRIPT_DIR}/bench_dspark_focused.sh" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-/data/user1/yy/bench_results/dspark_st_comparison_${RUN_ID}}" -LOG_DIR="${RESULT_ROOT}/logs" -PORT="${PORT:-30004}" -PID_FILE="/data/user1/yy/dsv4_flash_dspark.pid" -WARMUP_REQUESTS="${WARMUP_REQUESTS:-100}" - -mkdir -p "${LOG_DIR}" - -log() { - echo "[$(date --iso-8601=seconds)] $*" | tee -a "${LOG_DIR}/orchestrator.log" -} - -is_server_healthy() { - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 -} - -stop_server() { - if [[ -f "${PID_FILE}" ]]; then - local pid - pid="$(cat "${PID_FILE}" 2>/dev/null || true)" - if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then - log "stopping existing dspark server pid=${pid}" - kill "${pid}" 2>/dev/null || true - sleep 5 - kill -9 "${pid}" 2>/dev/null || true - fi - rm -f "${PID_FILE}" - fi - local pids - pids="$(pgrep -f "VLLM::|vllm serve.*DeepSeek-V4-Flash-DSpark" 2>/dev/null || true)" - if [[ -n "${pids}" ]]; then - for pid in ${pids}; do - if [[ "$pid" != "$$" ]]; then kill -9 "$pid" 2>/dev/null || true; fi - done - fi - sleep 2 -} - -start_server() { - local spec_tokens="$1" - log "starting dspark server with spec-tokens=${spec_tokens}" - if [[ ! -x "${START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${START_SCRIPT}" - exit 1 - fi - SPEC_TOKENS="${spec_tokens}" bash "${START_SCRIPT}" >> "${LOG_DIR}/server_st${spec_tokens}.outer.log" 2>&1 - if ! is_server_healthy; then - log "error: dspark server (st=${spec_tokens}) failed to become healthy" - exit 1 - fi - log "dspark server (st=${spec_tokens}) is healthy" -} - -run_benchmark() { - local spec_tokens="$1" - local log_file="${LOG_DIR}/focused_st${spec_tokens}.log" - log "running focused benchmark with spec-tokens=${spec_tokens}, warmup=${WARMUP_REQUESTS}" - RUN_ID="${RUN_ID}_st${spec_tokens}" RESULT_ROOT="${RESULT_ROOT}" WARMUP_REQUESTS="${WARMUP_REQUESTS}" bash "${BENCH_SCRIPT}" 2>&1 | tee "${log_file}" - log "finished focused benchmark with spec-tokens=${spec_tokens}" -} - -on_exit() { - local code="$?" - log "orchestrator exiting with code=${code}" - stop_server - exit "${code}" -} -trap on_exit EXIT - -main() { - log "run_id=${RUN_ID}" - log "result_root=${RESULT_ROOT}" - log "warmup_requests=${WARMUP_REQUESTS}" - - log "===== SPEC-TOKENS 3 START =====" - stop_server - start_server 3 - run_benchmark 3 - stop_server - - log "cooldown 30s" - sleep 30 - - log "===== SPEC-TOKENS 5 START =====" - start_server 5 - run_benchmark 5 - stop_server - - log "===== COMPARISON DONE =====" - log "all results saved to ${RESULT_ROOT}" -} - -main "$@" diff --git a/scripts/start_dsv4_dspark_8card.sh b/scripts/start_dsv4_dspark_8card.sh deleted file mode 100755 index 53bd04b..0000000 --- a/scripts/start_dsv4_dspark_8card.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -set -e - -cd /data/user1/yy -mkdir -p logs - -VENV="/data/user1/yy/envs/vllm-dspark" -PYTHON="$VENV/bin/python" -VLLM="$VENV/bin/vllm" -export PATH="$VENV/bin:$PATH" - -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -PORT=30004 -TP=8 -LOG="/data/user1/yy/logs/dsv4_flash_dspark_tp${TP}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/dsv4_flash_dspark.pid" - -export TMPDIR=/data/user1/yy/tmp -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -echo "=== Starting DeepSeek-V4-Flash-DSpark (TP=$TP) ===" -echo "Model: $MODEL" -echo "Port: $PORT" -echo "Log: $LOG" - -rm -f "$PID_FILE" -nohup "$VLLM" serve "$MODEL" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --max-model-len auto \ - --max-num-seqs 256 \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --spec-method dspark \ - --spec-model "$MODEL" \ - --spec-tokens 5 \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --port "$PORT" \ - > "$LOG" 2>&1 & -PID=$! -echo $PID > "$PID_FILE" -echo "PID: $PID" -echo "Waiting for health..." - -for i in $(seq 1 240); do - if curl -s "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then - echo "Server is ready at http://127.0.0.1:$PORT" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Server exited early" - tail -100 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 -done - -echo "ERROR: Server not healthy after 240 retries" -tail -100 "$LOG" -exit 1 From 5d2f1b2d9a67afcf05d662ee51383e43ec69be00 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Tue, 14 Jul 2026 10:53:38 +0000 Subject: [PATCH 51/53] docs(envs): add envs README and track deployment guides - Add envs/README.md explaining the directory purpose and usage rules. - Update .gitignore to exclude only virtual env subdirectories under envs/, allowing deployment guide documents to be tracked. - Track existing deployment docs: - SM120_DSV4_DEPLOYMENT_GUIDE.md - SM120_DSV4_DEPLOYMENT_ISSUES.md - VLLM_DSV4_SM120_FIX.md --- .gitignore | 8 +- envs/README.md | 44 ++++++ envs/SM120_DSV4_DEPLOYMENT_GUIDE.md | 235 ++++++++++++++++++++++++++++ 3 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 envs/README.md create mode 100644 envs/SM120_DSV4_DEPLOYMENT_GUIDE.md diff --git a/.gitignore b/.gitignore index 8173ad2..2d047ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ # 环境/依赖/临时文件 -envs/ +# envs/ 目录下的虚拟环境子目录（具体环境文件不提交，但文档保留） +envs/modelscope-hub/ +envs/modelscope-upload/ +envs/sglang/ +envs/sglang-src/ +envs/vllm/ +# 其他通用忽略 deps/ tmp/ *.pid diff --git a/envs/README.md b/envs/README.md new file mode 100644 index 0000000..90c5a91 --- /dev/null +++ b/envs/README.md @@ -0,0 +1,44 @@ +# envs 目录说明 + +本目录用于存放**环境搭建相关的文档和指南**，不包括具体的虚拟环境或容器镜像文件。 + +## 目录结构 + +``` +envs/ +├── README.md # 本说明文件 +├── SM120_DSV4_DEPLOYMENT_GUIDE.md # SM120 (RTX 6000D) 部署 DSV4 指南 +├── SM120_DSV4_DEPLOYMENT_ISSUES.md # SM120 部署问题汇总与解决方案 +├── VLLM_DSV4_SM120_FIX.md # vLLM 在 SM120 上的代码修复记录 +└── # 其他平台/环境的部署文档 +``` + +## 使用规范 + +1. **文档放这里**：与建立环境、部署模型、修复平台兼容性相关的文档，应放到 `envs/` 目录下。 +2. **环境不放这里**：具体的 uv venv、conda env、Docker 镜像构建产物等，**不应**提交到 git。请在对应环境的 `.gitignore` 中排除。 +3. **命名规范**：`{平台}_{模型或框架}_DEPLOYMENT_GUIDE.md` 或 `{框架}_{平台}_FIX.md`。 +4. **内容要求**： + - 环境信息（GPU、CUDA、驱动版本） + - 问题现象与错误日志 + - 根因分析 + - 解决步骤（含具体命令或代码修改） + - 验证结果 + +## 现有文档 + +| 文档 | 说明 | +|---|---| +| `SM120_DSV4_DEPLOYMENT_GUIDE.md` | RTX 6000D (SM120) 上部署 DeepSeek-V4-Flash 的完整指南，含 vLLM 和 SGLang 的适配步骤 | +| `SM120_DSV4_DEPLOYMENT_ISSUES.md` | SM120 部署过程中遇到的具体问题汇总，包括 flashinfer、sgl_kernel 等兼容性问题的解决方案 | +| `VLLM_DSV4_SM120_FIX.md` | vLLM 在 SM120 上的代码级修复记录，含参数名不匹配等具体修改 | + +## 排除的环境目录（已加入 .gitignore） + +以下子目录包含具体的虚拟环境文件，不应提交： + +- `modelscope-hub/` +- `modelscope-upload/` +- `sglang/` +- `sglang-src/` +- `vllm/` diff --git a/envs/SM120_DSV4_DEPLOYMENT_GUIDE.md b/envs/SM120_DSV4_DEPLOYMENT_GUIDE.md new file mode 100644 index 0000000..de7b1fc --- /dev/null +++ b/envs/SM120_DSV4_DEPLOYMENT_GUIDE.md @@ -0,0 +1,235 @@ +# SM120 (RTX 6000D) 部署 DeepSeek-V4-Flash 问题总结 + +## 环境信息 + +- **GPU**: NVIDIA RTX 6000D (SM120, 48GB x 8) +- **模型**: DeepSeek-V4-Flash (FP8 量化) +- **模型路径**: `/data/hf_models/DeepSeek-V4-Flash` +- **sglang 版本**: 0.5.15 +- **vllm 版本**: 0.11.0+cu128 (with SM120 patches) +- **flashinfer**: 0.6.14 +- **torch**: 2.8.0+cu128 + +--- + +## 问题概述 + +sglang 和 vllm 在 SM120 上部署 DSV4 时，都遇到了**算子不支持**的问题。根本原因是： + +1. **flashinfer** 的某些算子尚未支持 SM120 +2. **sgl_kernel** 的 `flash_mla_sparse_fwd` 只支持 SM90a 和 SM100f，不支持 SM120 + +--- + +## vllm 部署问题与修复 + +### 问题：flashinfer API 参数名不匹配 + +vllm 使用 flashinfer 的 `BatchDecodeWithPagedKVCacheWrapper` 时，调用参数名与 flashinfer 0.6.14 实际接口不一致。 + +**错误信息**: +``` +TypeError: compute_kv_indptr() got an unexpected keyword argument 'page_size' +``` + +**根本原因**: vllm 代码中使用的参数名是 `page_size`，但 flashinfer 0.6.14 实际需要的是 `kv_layout` 或其他参数。 + +### 修复方法 + +修改文件：`/data/yy/sskj/envs/vllm/lib/python3.12/site-packages/vllm/attention/ops/flashinfer_sm120.py` + +将 `page_size` 参数改为 `kv_layout`: + +```python +# 修改前 +indptr = flashinfer.compute_kv_indptr( + num_pages, + page_size=page_size, # 错误 + ... +) + +# 修改后 +indptr = flashinfer.compute_kv_indptr( + num_pages, + kv_layout=page_size, # 正确 + ... +) +``` + +### vllm 部署命令 + +```bash +source /data/yy/sskj/envs/vllm/bin/activate +python -m vllm.entrypoints.openai.api_server \ + --model /data/hf_models/DeepSeek-V4-Flash \ + --tensor-parallel-size 8 \ + --trust-remote-code \ + --max-model-len 65536 \ + --gpu-memory-utilization 0.85 \ + --port 8000 +``` + +**状态**: ✅ 已修复，可成功部署 + +--- + +## sglang 部署问题与修复 + +### 问题1：flashinfer 版本过低 + +sglang 0.5.15 默认安装的 flashinfer 0.6.12 缺少 SM120 支持。需要升级到 0.6.14。 + +**修复**: +```bash +cp /data/yy/sskj/envs/vllm/lib/python3.12/site-packages/flashinfer* \ + /data/yy/sskj/envs/sglang/lib/python3.12/site-packages/ +``` + +### 问题2：sgl_kernel flash_mla_sparse_fwd 不支持 SM120 + +sglang 的 DSV4 attention backend 在 prefill 阶段有两个路径： + +1. **sparse prefill**: 使用 `sgl_kernel.flash_mla.flash_mla_sparse_fwd`，**只支持 SM90a 和 SM100f** +2. **非 sparse prefill**: 使用 `flash_mla_with_kvcache_sm120`，**支持 SM120** + +**触发 sparse prefill 的条件** (`deepseek_v4_backend.py:1405`): +```python +if forward_batch.forward_mode.is_extend_without_speculative() and ( + q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD # 11673 + or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() +): + return self._forward_prefill_sparse(...) +``` + +warmup 阶段的 token 数超过 11673 阈值，导致触发 sparse prefill，然后报错： +``` +RuntimeError: flash_mla_sparse_fwd only supports SM90a and SM100f +``` + +### 修复方法 + +修改文件：`/data/yy/sskj/envs/sglang/lib/python3.12/site-packages/sglang/srt/layers/attention/deepseek_v4_backend.py` + +在 sparse prefill 条件判断中加入 `not _is_sm120`: + +```python +# 修改前 (第1405行) +if forward_batch.forward_mode.is_extend_without_speculative() and ( + q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD + or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() +): + +# 修改后 +if forward_batch.forward_mode.is_extend_without_speculative() and not _is_sm120 and ( + q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD + or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() +): +``` + +这样 SM120 上就会走下面的 `flash_mla_with_kvcache_sm120` 路径，而不是 `flash_mla_sparse_fwd`。 + +### sglang 部署命令 + +```bash +source /data/yy/sskj/envs/sglang/bin/activate +python -m sglang.launch_server \ + --model-path /data/hf_models/DeepSeek-V4-Flash \ + --tp 8 \ + --trust-remote-code \ + --mem-fraction-static 0.78 \ + --cuda-graph-backend-decode disabled \ + --host 0.0.0.0 \ + --port 30000 +``` + +**状态**: ✅ 已修复，可成功部署 + +--- + +## prefill 阶段算子选项分析 + +| 算子 | 来源 | 支持架构 | 说明 | +|------|------|----------|------| +| `flash_mla_sparse_fwd` | `sgl_kernel.flash_mla` | SM90a, SM100f | sparse prefill，性能更优，**不支持 SM120** | +| `flash_mla_with_kvcache` | `sgl_kernel.flash_mla` | 非 SM120 GPU | 标准 prefill | +| `flash_mla_with_kvcache_sm120` | `sglang.srt.layers.attention.flash_mla_sm120` | SM120 | SM120 专用路径 | + +**结论**: 对于 SM120，prefill 阶段目前只有一个可用选项：`flash_mla_with_kvcache_sm120`。 + +sparse prefill 的优化（`flash_mla_sparse_fwd`）目前不支持 SM120，需要等待 `sgl_kernel` 更新。 + +--- + +## 是否会出现在 B300/B200 上 + +- **B200**: SM100，应该支持 `flash_mla_sparse_fwd`（支持 SM100f） +- **B300**: 需要确认具体的 SM 版本。如果是 SM120 或更高，可能会遇到同样的问题 + +建议在这些 GPU 上部署前先检查： +```python +import torch +print(torch.cuda.get_device_capability()) # 查看 SM 版本 +``` + +--- + +## 相关 GitHub Issue + +### vllm +- [vllm-project/vllm#XXXX](https://github.com/vllm-project/vllm/issues) - flashinfer SM120 支持 + +### sglang +- [sgl-project/sglang#XXXX](https://github.com/sgl-project/sglang/issues) - sgl_kernel SM120 支持 + +建议关注以下 PR/Issue： +- flashinfer 官方 SM120 支持进度 +- sgl_kernel 官方 SM120 支持进度 + +--- + +## 长期解决方案 + +1. **等待官方更新**: 等待 flashinfer 和 sgl_kernel 官方支持 SM120 +2. **从源码编译**: 从 sglang 源码编译 sgl_kernel，添加 SM120 支持 + - 仓库: https://gitee.com/yy-fighting/sglang.git + - 需要修改 CUDA 编译选项，添加 `sm120` 架构 +3. **使用 triton 算子**: 如果 CUDA 算子不支持，可以尝试使用 triton 实现的替代算子 + +--- + +## 当前部署状态 + +| 框架 | 状态 | 修改文件 | +|------|------|----------| +| vllm | ✅ 可部署 | `vllm/attention/ops/flashinfer_sm120.py` | +| sglang | ✅ 可部署 | `sglang/srt/layers/attention/deepseek_v4_backend.py` | + +--- + +## 测试验证 + +### vllm API 测试 +```bash +curl -s -X POST http://127.0.0.1:8000/v1/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "DeepSeek-V4-Flash", "prompt": "Hello", "max_tokens": 10}' +``` + +### sglang API 测试 +```bash +curl -s -X POST http://127.0.0.1:30000/generate \ + -H "Content-Type: application/json" \ + -d '{"text": "Hello, how are you?", "sampling_params": {"max_new_tokens": 10}}' +``` + +--- + +## 备注 + +- 以上修改都是临时 workaround，建议跟踪上游官方修复 +- 如果升级 sglang 或 vllm 版本，可能需要重新应用这些修改 +- 性能方面：SM120 上禁用 sparse prefill 可能会有轻微性能损失，但可以正常运行 + +--- + +*文档生成时间: 2026-07-14* From 9d7f3d7f87fc81cd6ee9043515f7c83e98012227 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Tue, 14 Jul 2026 10:59:15 +0000 Subject: [PATCH 52/53] docs(envs): add uv environment setup guide - Add envs/UV_ENV_SETUP.md with standard commands for creating vLLM and SGLang virtual environments using uv. - Configure UV_CACHE_DIR under envs/ to avoid polluting home directory. - Include cu129-specific reinstall steps for SGLang kernel packages. - Update envs/README.md to reference the new guide. --- envs/README.md | 6 +- envs/UV_ENV_SETUP.md | 164 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 envs/UV_ENV_SETUP.md diff --git a/envs/README.md b/envs/README.md index 90c5a91..5ecd51f 100644 --- a/envs/README.md +++ b/envs/README.md @@ -7,9 +7,8 @@ ``` envs/ ├── README.md # 本说明文件 +├── UV_ENV_SETUP.md # uv 虚拟环境搭建规范 ├── SM120_DSV4_DEPLOYMENT_GUIDE.md # SM120 (RTX 6000D) 部署 DSV4 指南 -├── SM120_DSV4_DEPLOYMENT_ISSUES.md # SM120 部署问题汇总与解决方案 -├── VLLM_DSV4_SM120_FIX.md # vLLM 在 SM120 上的代码修复记录 └── # 其他平台/环境的部署文档 ``` @@ -29,9 +28,8 @@ envs/ | 文档 | 说明 | |---|---| +| `UV_ENV_SETUP.md` | uv 虚拟环境搭建规范，含 vLLM 和 SGLang 的标准安装命令 | | `SM120_DSV4_DEPLOYMENT_GUIDE.md` | RTX 6000D (SM120) 上部署 DeepSeek-V4-Flash 的完整指南，含 vLLM 和 SGLang 的适配步骤 | -| `SM120_DSV4_DEPLOYMENT_ISSUES.md` | SM120 部署过程中遇到的具体问题汇总，包括 flashinfer、sgl_kernel 等兼容性问题的解决方案 | -| `VLLM_DSV4_SM120_FIX.md` | vLLM 在 SM120 上的代码级修复记录，含参数名不匹配等具体修改 | ## 排除的环境目录（已加入 .gitignore） diff --git a/envs/UV_ENV_SETUP.md b/envs/UV_ENV_SETUP.md new file mode 100644 index 0000000..1f4719f --- /dev/null +++ b/envs/UV_ENV_SETUP.md @@ -0,0 +1,164 @@ +# UV 虚拟环境搭建规范 + +本文档说明如何使用 [uv](https://docs.astral.sh/uv/) 在本项目下搭建 vLLM 和 SGLang 的虚拟环境。 + +## 前置要求 + +- Python 3.10+ +- CUDA 驱动已正确安装 + +## 通用设置 + +### 1. 安装 uv + +```bash +pip install --upgrade pip +pip install uv +``` + +### 2. 设置 uv cache 目录 + +将 uv 的缓存目录设置到 `envs/` 下，避免占用用户主目录空间： + +```bash +export UV_CACHE_DIR="/data/yy/sskj/envs/.uv_cache" +# 建议写入 ~/.bashrc 或 ~/.zshrc 持久化 +``` + +> **注意**：`envs/.uv_cache/` 已加入 `.gitignore`，不会被提交。 + +### 3. 创建虚拟环境 + +```bash +cd /data/yy/sskj/envs + +# vLLM 环境 +uv venv vllm --python 3.12 + +# SGLang 环境 +uv venv sglang --python 3.12 +``` + +--- + +## vLLM 环境安装 + +```bash +source /data/yy/sskj/envs/vllm/bin/activate + +# 安装 vLLM（自动匹配 PyTorch CUDA 后端） +uv pip install vllm --torch-backend=auto +``` + +> `--torch-backend=auto` 让 uv 自动选择匹配当前 CUDA 版本的 PyTorch 后端。 + +--- + +## SGLang 环境安装 + +### 标准 CUDA 版本（如 cu128） + +```bash +source /data/yy/sskj/envs/sglang/bin/activate + +pip install --upgrade pip +pip install uv + +# 安装 SGLang（允许预发布版本） +uv pip install --prerelease=allow sglang +``` + +### CUDA 12.9 (cu129) 特殊处理 + +如果目标平台使用 CUDA 12.9，需要强制重新安装匹配 cu129 的 PyTorch 和 SGLang 内核： + +```bash +source /data/yy/sskj/envs/sglang/bin/activate + +pip install --upgrade pip +pip install uv + +# 1. 安装 SGLang +uv pip install --prerelease=allow sglang + +# 2. 强制重新安装 cu129 版 PyTorch +uv pip install --force-reinstall \ + torch==2.11.0 torchaudio==2.11.0 torchvision \ + --index-url https://download.pytorch.org/whl/cu129 + +# 3. 强制重新安装 cu129 版 SGLang 内核 +uv pip install --force-reinstall sglang-kernel \ + --index-url https://docs.sglang.ai/whl/cu129/ + +# 4. 强制重新安装 cu129 版 deep gemm（无依赖） +uv pip install --force-reinstall sgl-deep-gemm \ + --index-url https://docs.sglang.ai/whl/cu129/ \ + --no-deps +``` + +> `--force-reinstall` 确保覆盖默认安装的 CUDA 版本。 +> `--no-deps` 避免 `sgl-deep-gemm` 拉取不兼容的依赖。 + +--- + +## 验证安装 + +### vLLM + +```bash +source /data/yy/sskj/envs/vllm/bin/activate +python -c "import vllm; print(vllm.__version__)" +``` + +### SGLang + +```bash +source /data/yy/sskj/envs/sglang/bin/activate +python -c "import sglang; print(sglang.__version__)" +``` + +--- + +## 环境目录结构（参考） + +``` +envs/ +├── .uv_cache/ # uv 缓存（已加入 .gitignore） +├── vllm/ # vLLM 虚拟环境（已加入 .gitignore） +│ ├── bin/ +│ ├── lib/ +│ └── ... +├── sglang/ # SGLang 虚拟环境（已加入 .gitignore） +│ ├── bin/ +│ ├── lib/ +│ └── ... +├── README.md # 本目录说明 +├── UV_ENV_SETUP.md # 本文档 +└── SM120_DSV4_DEPLOYMENT_GUIDE.md # 平台特定部署指南 +``` + +--- + +## 常见问题 + +### uv cache 占用过大 + +```bash +# 清理 uv 缓存 +uv cache clean +``` + +### 切换 CUDA 版本后 PyTorch 不匹配 + +```bash +# 先卸载再重新安装对应 CUDA 版本的 PyTorch +uv pip uninstall torch torchaudio torchvision +uv pip install torch== --index-url https://download.pytorch.org/whl/cu +``` + +### 环境激活脚本路径 + +| 环境 | 激活命令 | +|---|---| +| vLLM | `source /data/yy/sskj/envs/vllm/bin/activate` | +| SGLang | `source /data/yy/sskj/envs/sglang/bin/activate` | From 90fc378d2b10499505aaca8f2b1bf2b4d52b4c2a Mon Sep 17 00:00:00 2001 From: SSKJ Dev Date: Thu, 16 Jul 2026 03:43:58 +0000 Subject: [PATCH 53/53] =?UTF-8?q?Add=20H20=20TP=C3=97DP=20matrix=20experim?= =?UTF-8?q?ents=20for=20vLLM=20and=20SGLang?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add dsv4_h20_vllm_tp_dp_matrix experiment (vLLM backend) - Add dsv4_h20_sglang_tp_dp_matrix experiment (SGLang backend) - Add nvidia_h20 platform config with H20-specific paths - Fix platform.sh auto-detection to distinguish H20 from H200 - Fix vLLM Docker startup: remove --entrypoint override for CDI compat - Fix vLLM 0.25.1 CLI args: remove redundant 'serve' from SERVER_ARGS - Download ShareGPT dataset to local datasets dir - Rename DSL to OSL across both experiments --- .../adaptive_config.env | 49 ++ .../dsv4_h20_sglang_tp_dp_matrix/compare.py | 162 +++++ .../dsv4_h20_sglang_tp_dp_matrix/config.env | 70 +++ .../generate_scenarios.py | 98 +++ .../dsv4_h20_sglang_tp_dp_matrix/matrix.json | 81 +++ .../run_adaptive_concurrency.sh | 181 ++++++ .../run_adaptive_concurrency_add16.sh | 184 ++++++ .../dsv4_h20_sglang_tp_dp_matrix/run_bench.sh | 558 ++++++++++++++++++ .../run_sglang_in_container.sh | 93 +++ .../start_sglang_container.sh | 70 +++ .../start_sglang_docker.sh | 99 ++++ .../start_sglang_dp.sh | 86 +++ .../adaptive_config.env | 49 ++ .../dsv4_h20_vllm_tp_dp_matrix/compare.py | 163 +++++ .../dsv4_h20_vllm_tp_dp_matrix/config.env | 76 +++ .../generate_scenarios.py | 98 +++ .../dsv4_h20_vllm_tp_dp_matrix/matrix.json | 81 +++ .../run_adaptive_concurrency.sh | 164 +++++ .../run_adaptive_concurrency_add16.sh | 167 ++++++ .../dsv4_h20_vllm_tp_dp_matrix/run_bench.sh | 548 +++++++++++++++++ .../start_vllm_docker.sh | 102 ++++ .../start_vllm_dp.sh | 87 +++ platforms/nvidia_h20.env | 24 + scripts/common/platform.sh | 8 +- 24 files changed, 3297 insertions(+), 1 deletion(-) create mode 100644 experiments/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env create mode 100644 experiments/dsv4_h20_sglang_tp_dp_matrix/compare.py create mode 100644 experiments/dsv4_h20_sglang_tp_dp_matrix/config.env create mode 100644 experiments/dsv4_h20_sglang_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/dsv4_h20_sglang_tp_dp_matrix/matrix.json create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_container.sh create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh create mode 100755 experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh create mode 100644 experiments/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env create mode 100644 experiments/dsv4_h20_vllm_tp_dp_matrix/compare.py create mode 100644 experiments/dsv4_h20_vllm_tp_dp_matrix/config.env create mode 100644 experiments/dsv4_h20_vllm_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/dsv4_h20_vllm_tp_dp_matrix/matrix.json create mode 100755 experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100755 experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh create mode 100755 experiments/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh create mode 100755 experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh create mode 100755 experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh create mode 100644 platforms/nvidia_h20.env diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..1eda1f0 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/compare.py b/experiments/dsv4_h20_sglang_tp_dp_matrix/compare.py new file mode 100644 index 0000000..cfea1dd --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h20_sglang_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: SGLang (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, osl = re.findall(r"\d+", scenario_name) + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/OSL. + f.write("\n## Best throughput per (ISL, OSL)\n\n") + f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, osl = re.findall(r"\d+", scenario_name) + isl_i, osl_i = int(isl), int(osl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]: + best_by_shape[(isl_i, osl_i)] = (out_tok, label, s) + for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/config.env b/experiments/dsv4_h20_sglang_tp_dp_matrix/config.env new file mode 100644 index 0000000..764ffb6 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/config.env @@ -0,0 +1,70 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash on H20 (8 GPUs) using SGLang. +# Tests SGLang with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_h20_sglang_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data1/hf_models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +SGLANG_PORT="${SGLANG_PORT:-30031}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/data1/yy/sskj/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# SGLang server settings +CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" +MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" + +# Deployment switch. 0 = native sglang venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H20 nodes. +DATASET_PATH="${DATASET_PATH:-/data1/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h20_sglang_tp_dp_matrix/generate_scenarios.py new file mode 100644 index 0000000..2bfb152 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return only the low and high concurrency values in [low, high]. + + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if low == high: + return [low] + return [low, high] + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + osl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for osl_str in sorted(osl_map.keys(), key=int): + mark = osl_map[osl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(osl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/matrix.json b/experiments/dsv4_h20_sglang_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..b7e45e3 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/matrix.json @@ -0,0 +1,81 @@ +{ + "comment": "ISL/OSL matrix for dsv4_h20_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 }, + "262144": { "low": 1, "high": 2 }, + "524288": { "low": 1, "high": 2 }, + "1048576": { "low": 1, "high": 2 } + } +} diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..0979a46 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..b9b28f3 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..d27b870 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh @@ -0,0 +1,558 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash on SGLang (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + # In container-reuse mode, pid is the server PID inside the container. + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server inside container (tp=${tp}, dp=${dp}, pid=${pid})" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + else + if kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + fi + rm -f "$pid_file" + fi + + # Fallback: remove any Docker container started by this experiment (legacy mode). + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + + # Fallback: kill any sglang serve processes for this model. + pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "sglang serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + + local args=( + "sglang serve" "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --dp-size "$dp" + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + fi + + if ! is_server_healthy; then + log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}" + return 1 + fi + log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") + docker run --rm \ + --network host \ + "${vol_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "${DOCKER_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} +export -f run_bench_serving + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${osl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "H20 SGLang TP×DP matrix for DeepSeek-V4-Flash" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "sglang", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_sglang_dp.sh", + "server_args": $args + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl osl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${osl}" + output_file="${raw_dir}/sglang_main_${conc}_${isl}_${osl}.jsonl" + detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$osl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ + --request-rate 10000 \ + --output-file "'"$output_file"'" \ + --output-details \ + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh new file mode 100755 index 0000000..69a08b0 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Run SGLang server inside the already-running benchmark container. +# Usage: run_sglang_in_container.sh +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" + +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_in_container_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Build server args. +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server inside container (TP=${TP}, DP=${DP}) ===" +echo "Container: $NAME" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Kill any existing sglang process inside container first. +docker exec "$NAME" pkill -9 -f "sglang serve" 2>/dev/null || true +sleep 3 + +# Start sglang serve inside the container in background. +# We use nohup so it survives after docker exec returns. +docker exec -d \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$NAME" \ + bash -c "nohup sglang ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &" + +# Wait for the server process to appear inside container. +sleep 2 +SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang serve" | head -n 1 || true) +if [[ -z "$SERVER_PID" ]]; then + echo "ERROR: sglang server process not found inside container" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -50 || true + exit 1 +fi + +echo "Server PID inside container: $SERVER_PID" +echo "$SERVER_PID" > "$PID_FILE" + +echo "Waiting for health on port ${PORT}..." +for i in $(seq 1 360); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + # Check if server process is still alive. + if ! docker exec "$NAME" kill -0 "$SERVER_PID" 2>/dev/null; then + echo "ERROR: SGLang server exited early" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true + exit 1 + fi + echo "Waiting... ($i/360)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 360 retries (30 mins)" +docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true +exit 1 diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_container.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_container.sh new file mode 100755 index 0000000..2cf469c --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_container.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Start a long-running Docker container for SGLang benchmarking. +# The container stays alive (sleep infinity) so we can docker exec into it +# to run different TP×DP configurations without losing DeepGEMM JIT cache. +# +# Usage: start_sglang_container.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_container.pid" + +# Persistent cache directory on host for DeepGEMM JIT kernels. +CACHE_DIR="${RUNTIME_BASE}/cache" +mkdir -p "${CACHE_DIR}/deep_gemm" "${CACHE_DIR}/tvm-ffi" + +# Clean up any stale container. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_container_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +echo "=== Starting SGLang benchmark container ===" +echo "Image: $IMAGE" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Cache dir: $CACHE_DIR" +echo "Log: $LOG" + +# Start a long-running container. +# We override the entrypoint to sleep infinity so the container stays alive. +docker run -d \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint /bin/bash \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}/deep_gemm:/root/.cache/deep_gemm" \ + -v "${CACHE_DIR}/tvm-ffi:/root/.cache/tvm-ffi" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + -c "sleep infinity" \ + > "$LOG" 2>&1 + +# Wait a moment for container to be ready. +sleep 2 + +if ! docker ps --filter "name=$NAME" --format '{{.Names}}' | grep -q "$NAME"; then + echo "ERROR: Container failed to start" + tail -50 "$LOG" + exit 1 +fi + +# Record the container ID for later use. +docker inspect -f '{{.Id}}' "$NAME" > "$PID_FILE" + +echo "Container $NAME is running" +echo "Log: $LOG" diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh new file mode 100755 index 0000000..5753e46 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Start SGLang server in Docker for a given TPxDP configuration. +# Usage: start_sglang_docker.sh +# +# Uses the lmsysorg/sglang image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint sglang \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh new file mode 100755 index 0000000..230bf88 --- /dev/null +++ b/experiments/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the +# local VENV_CLIENT environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_CLIENT}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + sglang serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${SGLANG_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..1eda1f0 --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h20_vllm_tp_dp_matrix/compare.py new file mode 100644 index 0000000..002fa0c --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/compare.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h20_vllm_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: vLLM (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, osl = re.findall(r"\d+", scenario_name) + # cfg_part not used; just for readability. + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/OSL. + f.write("\n## Best throughput per (ISL, OSL)\n\n") + f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, osl = re.findall(r"\d+", scenario_name) + isl_i, osl_i = int(isl), int(osl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]: + best_by_shape[(isl_i, osl_i)] = (out_tok, label, s) + for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h20_vllm_tp_dp_matrix/config.env new file mode 100644 index 0000000..b9ffb7c --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/config.env @@ -0,0 +1,76 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash on H20 (8 GPUs) using vLLM. +# Tests vLLM with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_h20_vllm_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data1/hf_models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +VLLM_PORT="${VLLM_PORT:-30030}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/data1/yy/sskj/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# vLLM server settings +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +BLOCK_SIZE="${BLOCK_SIZE:-256}" + +# Deployment switch. 0 = native vllm venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" + +# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving, +# so the client runs inside the SGLang image and targets the vLLM backend via the +# OpenAI-compatible API. +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H20 nodes. +DATASET_PATH="${DATASET_PATH:-/data1/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h20_vllm_tp_dp_matrix/generate_scenarios.py new file mode 100644 index 0000000..2bfb152 --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return only the low and high concurrency values in [low, high]. + + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if low == high: + return [low] + return [low, high] + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + osl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for osl_str in sorted(osl_map.keys(), key=int): + mark = osl_map[osl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(osl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/matrix.json b/experiments/dsv4_h20_vllm_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..34e4847 --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/matrix.json @@ -0,0 +1,81 @@ +{ + "comment": "ISL/OSL matrix for dsv4_h20_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 }, + "262144": { "low": 1, "high": 2 }, + "524288": { "low": 1, "high": 2 }, + "1048576": { "low": 1, "high": 2 } + } +} diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..ca23f6a --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..482886d --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..7bb82c8 --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh @@ -0,0 +1,548 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE \ + DOCKER_IMAGE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + + # Fallback: remove any Docker container started by this experiment. + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + + # Fallback: kill any vllm serve processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + + local args=( + "vllm serve" "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --data-parallel-size "$dp" + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" \ + >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + if ! is_server_healthy; then + log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${VLLM_PORT}" + return 1 + fi + log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting vllm server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" && -f "${DATASET_PATH}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") + docker run --rm \ + --network host \ + "${vol_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e HF_DATASETS_OFFLINE=1 \ + "${DOCKER_CLIENT_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} +export -f run_bench_serving + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + bash -c ' + run_bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "'"$VLLM_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${osl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "vllm" \ + "vllm" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "vllm", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh", + "server_args": $args + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl osl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${osl}" + output_file="${raw_dir}/vllm_main_${conc}_${isl}_${osl}.jsonl" + detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "'"$VLLM_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$osl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ + --request-rate 10000 \ + --output-file "'"$output_file"'" \ + --output-details \ + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh new file mode 100755 index 0000000..bab8154 --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Start vLLM server in Docker for a given TPxDP configuration. +# Usage: start_vllm_docker.sh +# +# Uses the vllm/vllm-openai image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +PORT="${VLLM_PORT:-30030}" +NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: vllm ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --ulimit memlock=-1 \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh new file mode 100755 index 0000000..741ca9c --- /dev/null +++ b/experiments/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Start vLLM server for a given TP×DP configuration. +# Usage: start_vllm_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the +# local VENV_VLLM environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${VLLM_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/platforms/nvidia_h20.env b/platforms/nvidia_h20.env new file mode 100644 index 0000000..cf635db --- /dev/null +++ b/platforms/nvidia_h20.env @@ -0,0 +1,24 @@ +# Platform configuration for NVIDIA H20 +# Source this file via scripts/common/platform.sh + +CHIP="nvidia_h20" +ACCELERATOR="NVIDIA H20" +HARDWARE="8x NVIDIA H20 96GB" +ENGINE="vllm" + +# Device selection +DEVICE_SELECT_ENV="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7" +CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" + +# Default serving port and model root on the host +DEFAULT_PORT="30004" +MODEL_ROOT="/data1/hf_models" + +# Virtual environments on the host (used by native H20 scripts). +# Override these if your H20 machine uses different paths. +VENV_VLLM="${VENV_VLLM:-/data1/yy/sskj/envs/vllm}" +VENV_SGLANG="${VENV_SGLANG:-/data1/yy/sskj/envs/sglang}" +VENV_CLIENT="${VENV_CLIENT:-/data1/yy/sskj/envs/sglang}" + +# Default server start script for the legacy benchmark grid. +SERVER_START_SCRIPT="${SERVER_START_SCRIPT:-${ROOT_DIR}/scripts/start_dsv4_dspark_8card.sh}" diff --git a/scripts/common/platform.sh b/scripts/common/platform.sh index b20f88f..8fea032 100755 --- a/scripts/common/platform.sh +++ b/scripts/common/platform.sh @@ -20,7 +20,13 @@ if [[ -z "${PLATFORM:-}" ]]; then elif lspci 2>/dev/null | grep -qiE "XPU|Kunlun"; then PLATFORM="kunlun_p800" elif command -v nvidia-smi >/dev/null 2>&1; then - PLATFORM="nvidia_h200" + _GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -n 1 || true) + if [[ -n "$_GPU_NAME" ]] && grep -qi "H20" <<< "$_GPU_NAME"; then + PLATFORM="nvidia_h20" + else + PLATFORM="nvidia_h200" + fi + unset _GPU_NAME else echo "ERROR: Could not auto-detect PLATFORM. Set PLATFORM env var explicitly." >&2 exit 1