From b1b5f50f56106041632c54df41d657bdc9ebbaf0 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 04:08:40 +0000 Subject: [PATCH] results(p800): max context length exploration run 20260709-033134 P800 SGLang INT8 max supported input length: 131072 tokens. 262144 tokens fails during warmup (server connection lost, likely OOM). --- .../results/20260709-033134/report.md | 24 ++ .../results/20260709-033134/results.json | 315 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md create mode 100644 experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md b/experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md new file mode 100644 index 0000000..dfc6670 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-033134/report.md @@ -0,0 +1,24 @@ +# P800 Max Context Length Exploration Report + +- Result root: `/data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134` +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Hardware: 8x Kunlun P800 XPU +- TP: 8, EP: 8 +- Output len: 1, num prompts: 1, concurrency: 1 + +## Per-attempt results + +| Target len | Max context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4096 | 5120 | ✅ pass | 233.71 | 0.00 | 233.72 | 233.72 | | +| 8192 | 9216 | ✅ pass | 239.86 | 0.00 | 239.87 | 239.87 | | +| 16384 | 17408 | ✅ pass | 262.19 | 0.00 | 262.20 | 262.20 | | +| 32768 | 33792 | ✅ pass | 309.52 | 0.00 | 309.54 | 309.54 | | +| 65536 | 66560 | ✅ pass | 390.71 | 0.00 | 390.72 | 390.72 | | +| 131072 | 132096 | ✅ pass | 583.94 | 0.00 | 583.95 | 583.95 | | +| 262144 | 263168 | ❌ fail | 0.00 | 0.00 | 0.00 | 0.00 | bench_serving exited with code 137; see /data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134/logs/sglang_ctx_262144.log | + +## Maximum supported input length + +- **sglang-xpu**: 131072 tokens + diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json b/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json new file mode 100644 index 0000000..41ea383 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json @@ -0,0 +1,315 @@ +{ + "metadata": { + "experiment": "dsv4_p800_max_context_length", + "run_id": "20260709-033134", + "timestamp": "2026-07-09T03:31:34+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_max_context_length/run_bench.sh", + "env": "", + "git_commit": "407bc21", + "git_dirty": "dirty", + "description": "P800 max context length exploration for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024 + }, + "scenarios": [ + { + "backend": "sglang", + "target_len": 4096, + "max_context_len": 5120, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 5120 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 5120", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.3655072329565883, + "request_throughput": 0.2971320311564159, + "input_token_throughput": 1217.0527996166795, + "output_token_throughput": 0.2971320311564159, + "total_input_tokens": 4096, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 233.7192000122741, + "p50": 233.7192000122741, + "p90": 233.7192000122741, + "p95": 0.0, + "p99": 233.7192000122741 + }, + "ttft_ms": { + "mean": 233.70750999310985, + "p50": 233.70750999310985, + "p90": 0.0, + "p95": 0.0, + "p99": 233.70750999310985 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 8192, + "max_context_len": 9216, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 9216 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 9216", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.376425461960025, + "request_throughput": 0.2961712056926312, + "input_token_throughput": 2426.234517034035, + "output_token_throughput": 0.2961712056926312, + "total_input_tokens": 8192, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 239.87414099974558, + "p50": 239.87414099974558, + "p90": 239.87414099974558, + "p95": 0.0, + "p99": 239.87414099974558 + }, + "ttft_ms": { + "mean": 239.86148100811988, + "p50": 239.86148100811988, + "p90": 0.0, + "p95": 0.0, + "p99": 239.86148100811988 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 16384, + "max_context_len": 17408, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 17408 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 17408", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.4154617289896123, + "request_throughput": 0.292786182176261, + "input_token_throughput": 4797.00880877586, + "output_token_throughput": 0.292786182176261, + "total_input_tokens": 16384, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 262.1981970150955, + "p50": 262.1981970150955, + "p90": 262.1981970150955, + "p95": 0.0, + "p99": 262.1981970150955 + }, + "ttft_ms": { + "mean": 262.1856770128943, + "p50": 262.1856770128943, + "p90": 0.0, + "p95": 0.0, + "p99": 262.1856770128943 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 32768, + "max_context_len": 33792, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 33792 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 33792", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.431973898957949, + "request_throughput": 0.2913775073591411, + "input_token_throughput": 9547.858161144335, + "output_token_throughput": 0.2913775073591411, + "total_input_tokens": 32768, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 309.53772098291665, + "p50": 309.53772098291665, + "p90": 309.53772098291665, + "p95": 0.0, + "p99": 309.53772098291665 + }, + "ttft_ms": { + "mean": 309.5244810101576, + "p50": 309.5244810101576, + "p90": 0.0, + "p95": 0.0, + "p99": 309.5244810101576 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 65536, + "max_context_len": 66560, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 66560 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 66560", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.5316250980249606, + "request_throughput": 0.28315576320919333, + "input_token_throughput": 18556.896097677694, + "output_token_throughput": 0.28315576320919333, + "total_input_tokens": 65536, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 390.7171559985727, + "p50": 390.7171559985727, + "p90": 390.7171559985727, + "p95": 0.0, + "p99": 390.7171559985727 + }, + "ttft_ms": { + "mean": 390.70626598550007, + "p50": 390.70626598550007, + "p90": 0.0, + "p95": 0.0, + "p99": 390.70626598550007 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 131072, + "max_context_len": 132096, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 132096 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 132096", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 0.6164236619952135, + "request_throughput": 1.6222608923921629, + "input_token_throughput": 212632.97968762557, + "output_token_throughput": 1.6222608923921629, + "total_input_tokens": 131072, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 583.9470439823344, + "p50": 583.9470439823344, + "p90": 583.9470439823344, + "p95": 0.0, + "p99": 583.9470439823344 + }, + "ttft_ms": { + "mean": 583.9353539631702, + "p50": 583.9353539631702, + "p90": 0.0, + "p95": 0.0, + "p99": 583.9353539631702 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + }, + { + "backend": "sglang", + "target_len": 262144, + "max_context_len": 263168, + "success": false, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 263168 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 263168", + "metrics": null, + "error": "bench_serving exited with code 137; see /data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134/logs/sglang_ctx_262144.log" + } + ] +} \ No newline at end of file