From 8562c265240da6cb749e807900367bda935e3957 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 20 Jul 2026 16:05:33 +0800 Subject: [PATCH] fix(sglang): cap SM120 CUDA graph prefill --- README.md | 1 + .../pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env | 9 +++++++++ .../run_adaptive_concurrency.sh | 4 +++- .../run_adaptive_concurrency_add16.sh | 4 +++- .../start_sglang_docker.sh | 5 +++++ .../dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh | 7 +++++-- 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d912ef9..5e3f2af 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ > - 仅用于验证 Gitee 受保护分支的自动 Pull Request 创建与评审流程；不修改实验代码、配置或结果口径。 > **更新（2026-07-20，`903084c`）** +> - RTX 6000D 的 SGLang DSV4-Flash 默认传入 `--cuda-graph-max-bs-decode 16`，避免 SM120 sparse MLA 在大 batch CUDA Graph 预捕获时 OOM。该参数仅限制预捕获图的 decode batch 上限，不限制服务的 `max-running-requests` 或 benchmark 并发；其他平台默认不受影响。 > - H20 / RTX 6000D 的 TPxDP matrix baseline 不再显式限制模型上下文或服务内并发：vLLM 使用默认 `max-model-len` / `max-num-seqs`，SGLang 使用默认 `context-length` / `max-running-requests`。 > - SGLang 固定使用 `--moe-runner-backend marlin`；vLLM 保持框架默认 MoE backend。 > - `run_adaptive_concurrency_add16.sh` 在首点 OOM 时会重启服务并按 `C=16 -> 8 -> 1` 回退；仅 H20 / RTX 6000D matrix 默认开启，H200 / P800 保持原行为。 diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env index 7d3ba85..587e946 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -38,6 +38,10 @@ declare -a PARALLEL_CONFIGS=( MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" +# RTX 6000D cannot capture DSV4 decode graphs at the default large batches. +# This limits graph pre-capture only; it does not cap request concurrency. +CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-16}" + # Deployment switch. 0 = native sglang venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" @@ -69,3 +73,8 @@ DRY_RUN="${DRY_RUN:-0}" # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. GRID_LIMIT="${GRID_LIMIT:-0}" + +# PyTorch CUDA allocator setting for the SGLang server. expandable_segments +# reduces fragmentation from the GiB-scale indexer temporaries that OOM the +# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096. +PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 4be536a..286d9b9 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -79,6 +79,9 @@ engine_build_server_args() { if (( dp > 1 )); then args+=(--dp-size "$dp") fi + if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then + args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") + fi printf '%q ' "${args[@]}" } @@ -149,7 +152,6 @@ engine_run_bench() { --output-file "$output_file" --output-details --disable-tqdm - --disable-stream ) if [[ "$BENCH_DATASET_NAME" == "random" ]]; then bench_args+=(--dataset-path "$DATASET_PATH") diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 1be9866..ec77610 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -79,6 +79,9 @@ engine_build_server_args() { if (( dp > 1 )); then args+=(--dp-size "$dp") fi + if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then + args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") + fi printf '%q ' "${args[@]}" } @@ -149,7 +152,6 @@ engine_run_bench() { --output-file "$output_file" --output-details --disable-tqdm - --disable-stream ) if [[ "$BENCH_DATASET_NAME" == "random" ]]; then bench_args+=(--dataset-path "$DATASET_PATH") diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh index fe758aa..23d0901 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -44,6 +44,10 @@ if [[ "$DP" -gt 1 ]]; then ) fi +if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then + SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") +fi + SERVER_ARGS_STR="${SERVER_ARGS[*]}" echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" @@ -73,6 +77,7 @@ nohup docker run --rm \ -e PYTHONUNBUFFERED=1 \ -e HF_HUB_OFFLINE=1 \ -e TRANSFORMERS_OFFLINE=1 \ + -e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \ "$IMAGE" \ "${SERVER_ARGS[@]}" \ > "$LOG" 2>&1 & diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh index 3472823..ccbd125 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -24,6 +24,7 @@ mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" VENV="${VENV_CLIENT}" export PATH="$VENV/bin:$PATH" export PYTHONUNBUFFERED=1 +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" export TMPDIR="${RUNTIME_BASE}/tmp" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" @@ -38,8 +39,6 @@ SERVER_ARGS=( --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$SGLANG_PORT" @@ -51,6 +50,10 @@ if [[ "$DP" -gt 1 ]]; then ) fi +if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then + SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") +fi + SERVER_ARGS_STR="${SERVER_ARGS[*]}" echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ==="