qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134/results.json
Quantong Qiu b1b5f50f56 results(p800): max context length exploration run 20260709-033134 
P800 SGLang INT8 max supported input length: 131072 tokens.
262144 tokens fails during warmup (server connection lost, likely OOM).
2026-07-09 04:08:40 +00:00

315 lines
14 KiB
JSON
Raw Blame History

{
"metadata": {
"experiment": "dsv4_p800_max_context_length",
"run_id": "20260709-033134",
"timestamp": "2026-07-09T03:31:34+00:00",
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
"backend": "sglang",
"engine": "sglang-xpu",
"hardware": "8x Kunlun P800 XPU",
"accelerator": "Kunlun P800 XPU",
"chip": "kunlun_p800",
"script": "experiments/dsv4_p800_max_context_length/run_bench.sh",
"env": "",
"git_commit": "407bc21",
"git_dirty": "dirty",
"description": "P800 max context length exploration for DeepSeek-V4-Flash-INT8"
},
"config": {
"tp": 8,
"ep": 8,
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
"output_len": 1,
"num_prompts": 1,
"max_concurrency": 1,
"request_rate": 10000,
"context_pad": 1024
},
"scenarios": [
{
"backend": "sglang",
"target_len": 4096,
"max_context_len": 5120,
"success": true,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 5120 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 5120",
"metrics": {
"success_count": 1,
"total_count": 1,
"duration_s": 3.3655072329565883,
"request_throughput": 0.2971320311564159,
"input_token_throughput": 1217.0527996166795,
"output_token_throughput": 0.2971320311564159,
"total_input_tokens": 4096,
"total_output_tokens": 1,
"e2e_ms": {
"mean": 233.7192000122741,
"p50": 233.7192000122741,
"p90": 233.7192000122741,
"p95": 0.0,
"p99": 233.7192000122741
},
"ttft_ms": {
"mean": 233.70750999310985,
"p50": 233.70750999310985,
"p90": 0.0,
"p95": 0.0,
"p99": 233.70750999310985
},
"tpot_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
},
"itl_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
}
},
"error": null
},
{
"backend": "sglang",
"target_len": 8192,
"max_context_len": 9216,
"success": true,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 9216 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 9216",
"metrics": {
"success_count": 1,
"total_count": 1,
"duration_s": 3.376425461960025,
"request_throughput": 0.2961712056926312,
"input_token_throughput": 2426.234517034035,
"output_token_throughput": 0.2961712056926312,
"total_input_tokens": 8192,
"total_output_tokens": 1,
"e2e_ms": {
"mean": 239.87414099974558,
"p50": 239.87414099974558,
"p90": 239.87414099974558,
"p95": 0.0,
"p99": 239.87414099974558
},
"ttft_ms": {
"mean": 239.86148100811988,
"p50": 239.86148100811988,
"p90": 0.0,
"p95": 0.0,
"p99": 239.86148100811988
},
"tpot_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
},
"itl_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
}
},
"error": null
},
{
"backend": "sglang",
"target_len": 16384,
"max_context_len": 17408,
"success": true,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 17408 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 17408",
"metrics": {
"success_count": 1,
"total_count": 1,
"duration_s": 3.4154617289896123,
"request_throughput": 0.292786182176261,
"input_token_throughput": 4797.00880877586,
"output_token_throughput": 0.292786182176261,
"total_input_tokens": 16384,
"total_output_tokens": 1,
"e2e_ms": {
"mean": 262.1981970150955,
"p50": 262.1981970150955,
"p90": 262.1981970150955,
"p95": 0.0,
"p99": 262.1981970150955
},
"ttft_ms": {
"mean": 262.1856770128943,
"p50": 262.1856770128943,
"p90": 0.0,
"p95": 0.0,
"p99": 262.1856770128943
},
"tpot_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
},
"itl_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
}
},
"error": null
},
{
"backend": "sglang",
"target_len": 32768,
"max_context_len": 33792,
"success": true,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 33792 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 33792",
"metrics": {
"success_count": 1,
"total_count": 1,
"duration_s": 3.431973898957949,
"request_throughput": 0.2913775073591411,
"input_token_throughput": 9547.858161144335,
"output_token_throughput": 0.2913775073591411,
"total_input_tokens": 32768,
"total_output_tokens": 1,
"e2e_ms": {
"mean": 309.53772098291665,
"p50": 309.53772098291665,
"p90": 309.53772098291665,
"p95": 0.0,
"p99": 309.53772098291665
},
"ttft_ms": {
"mean": 309.5244810101576,
"p50": 309.5244810101576,
"p90": 0.0,
"p95": 0.0,
"p99": 309.5244810101576
},
"tpot_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
},
"itl_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
}
},
"error": null
},
{
"backend": "sglang",
"target_len": 65536,
"max_context_len": 66560,
"success": true,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 66560 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 66560",
"metrics": {
"success_count": 1,
"total_count": 1,
"duration_s": 3.5316250980249606,
"request_throughput": 0.28315576320919333,
"input_token_throughput": 18556.896097677694,
"output_token_throughput": 0.28315576320919333,
"total_input_tokens": 65536,
"total_output_tokens": 1,
"e2e_ms": {
"mean": 390.7171559985727,
"p50": 390.7171559985727,
"p90": 390.7171559985727,
"p95": 0.0,
"p99": 390.7171559985727
},
"ttft_ms": {
"mean": 390.70626598550007,
"p50": 390.70626598550007,
"p90": 0.0,
"p95": 0.0,
"p99": 390.70626598550007
},
"tpot_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
},
"itl_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
}
},
"error": null
},
{
"backend": "sglang",
"target_len": 131072,
"max_context_len": 132096,
"success": true,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 132096 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 132096",
"metrics": {
"success_count": 1,
"total_count": 1,
"duration_s": 0.6164236619952135,
"request_throughput": 1.6222608923921629,
"input_token_throughput": 212632.97968762557,
"output_token_throughput": 1.6222608923921629,
"total_input_tokens": 131072,
"total_output_tokens": 1,
"e2e_ms": {
"mean": 583.9470439823344,
"p50": 583.9470439823344,
"p90": 583.9470439823344,
"p95": 0.0,
"p99": 583.9470439823344
},
"ttft_ms": {
"mean": 583.9353539631702,
"p50": 583.9353539631702,
"p90": 0.0,
"p95": 0.0,
"p99": 583.9353539631702
},
"tpot_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
},
"itl_ms": {
"mean": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0
}
},
"error": null
},
{
"backend": "sglang",
"target_len": 262144,
"max_context_len": 263168,
"success": false,
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 263168 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 263168",
"metrics": null,
"error": "bench_serving exited with code 137; see /data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-033134/logs/sglang_ctx_262144.log"
}
]
}
Reference in New Issue View Git Blame Copy Permalink