From 46e05a57a4c42894dc676d4142f04113f5fc5401 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Thu, 9 Jul 2026 05:13:04 +0000 Subject: [PATCH] Re-enable chunked prefill for P800 long-context matrix stability Without chunked prefill the server OOM/crashes (exit 137) during or immediately after concurrent 64k-context scenarios. Speculative decoding remains disabled as requested. --- experiments/dsv4_p800_long_context_matrix/start_sglang.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh index 5812801..4489ace 100755 --- a/experiments/dsv4_p800_long_context_matrix/start_sglang.sh +++ b/experiments/dsv4_p800_long_context_matrix/start_sglang.sh @@ -92,8 +92,9 @@ env_args=( # Launch args. Keep the proven P800 INT8 command from dsv4_p800_max_context_length, # but make --context-length and --max-running-requests dynamic per group. -# Note: disable chunked prefill and speculative decoding for the long-context matrix. -launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" +# Note: chunked prefill is required for stable concurrent long-context prefill on P800; +# speculative decoding is disabled as requested. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests ${MAX_RUNNING} --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" # Base64-encode the bootstrap command to avoid host-shell quoting issues. server_cmd=$(cat <