qqtang/sskj
1
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: cuda-graph-max-bs=64 (256 OOMs during capture)

Browse Source
This commit is contained in:
Zhiyi Hong 2026-07-28 17:31:52 +08:00
parent cd362c5ca0
commit 6d3338244b
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
experiments/pro6000/dsv4_pro6000_sglang_tp16/config.env
View File

@ -35,7 +35,7 @@ NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
# max_running_requests must >= max concurrency tested (128) to avoid queueing # max_running_requests must >= max concurrency tested (128) to avoid queueing
# cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf # cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-256}" CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-64}"
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}" MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}"
# ---- Benchmark ---- # ---- Benchmark ----