fix(sglang): cap SM120 CUDA graph prefill

README.md

@@ -4,6 +4,7 @@
 > - 仅用于验证 Gitee 受保护分支的自动 Pull Request 创建与评审流程；不修改实验代码、配置或结果口径。
 
 > **更新（2026-07-20，`903084c`）**
+> - RTX 6000D 的 SGLang DSV4-Flash 默认传入 `--cuda-graph-max-bs-decode 16`，避免 SM120 sparse MLA 在大 batch CUDA Graph 预捕获时 OOM。该参数仅限制预捕获图的 decode batch 上限，不限制服务的 `max-running-requests` 或 benchmark 并发；其他平台默认不受影响。
 > - H20 / RTX 6000D 的 TPxDP matrix baseline 不再显式限制模型上下文或服务内并发：vLLM 使用默认 `max-model-len` / `max-num-seqs`，SGLang 使用默认 `context-length` / `max-running-requests`。
 > - SGLang 固定使用 `--moe-runner-backend marlin`；vLLM 保持框架默认 MoE backend。
 > - `run_adaptive_concurrency_add16.sh` 在首点 OOM 时会重启服务并按 `C=16 -> 8 -> 1` 回退；仅 H20 / RTX 6000D matrix 默认开启，H200 / P800 保持原行为。

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env

@@ -38,6 +38,10 @@ declare -a PARALLEL_CONFIGS=(
 MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}"
 MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"
 
+# RTX 6000D cannot capture DSV4 decode graphs at the default large batches.
+# This limits graph pre-capture only; it does not cap request concurrency.
+CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-16}"
+
 # Deployment switch. 0 = native sglang venv, 1 = Docker.
 USE_DOCKER="${USE_DOCKER:-1}"
 DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
@@ -69,3 +73,8 @@ DRY_RUN="${DRY_RUN:-0}"
 
 # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
 GRID_LIMIT="${GRID_LIMIT:-0}"
+
+# PyTorch CUDA allocator setting for the SGLang server. expandable_segments
+# reduces fragmentation from the GiB-scale indexer temporaries that OOM the
+# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096.
+PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -79,6 +79,9 @@ engine_build_server_args() {
   if (( dp > 1 )); then
     args+=(--dp-size "$dp")
   fi
+  if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
+    args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
+  fi
   printf '%q ' "${args[@]}"
 }
 
@@ -149,7 +152,6 @@ engine_run_bench() {
     --output-file "$output_file"
     --output-details
     --disable-tqdm
-    --disable-stream
   )
   if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
     bench_args+=(--dataset-path "$DATASET_PATH")

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -79,6 +79,9 @@ engine_build_server_args() {
   if (( dp > 1 )); then
     args+=(--dp-size "$dp")
   fi
+  if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
+    args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
+  fi
   printf '%q ' "${args[@]}"
 }
 
@@ -149,7 +152,6 @@ engine_run_bench() {
     --output-file "$output_file"
     --output-details
     --disable-tqdm
-    --disable-stream
   )
   if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
     bench_args+=(--dataset-path "$DATASET_PATH")

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh

@@ -44,6 +44,10 @@ if [[ "$DP" -gt 1 ]]; then
   )
 fi
 
+if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
+  SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
+fi
+
 SERVER_ARGS_STR="${SERVER_ARGS[*]}"
 
 echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ==="
@@ -73,6 +77,7 @@ nohup docker run --rm \
   -e PYTHONUNBUFFERED=1 \
   -e HF_HUB_OFFLINE=1 \
   -e TRANSFORMERS_OFFLINE=1 \
+  -e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \
   "$IMAGE" \
   "${SERVER_ARGS[@]}" \
   > "$LOG" 2>&1 &

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh

@@ -24,6 +24,7 @@ mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
 VENV="${VENV_CLIENT}"
 export PATH="$VENV/bin:$PATH"
 export PYTHONUNBUFFERED=1
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
 export TMPDIR="${RUNTIME_BASE}/tmp"
 export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
 
@@ -38,8 +39,6 @@ SERVER_ARGS=(
   --trust-remote-code
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
-  --context-length "$CONTEXT_LENGTH"
-  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
   --host 0.0.0.0
   --port "$SGLANG_PORT"
@@ -51,6 +50,10 @@ if [[ "$DP" -gt 1 ]]; then
   )
 fi
 
+if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
+  SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
+fi
+
 SERVER_ARGS_STR="${SERVER_ARGS[*]}"
 
 echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ==="