diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp16/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp16/config.env index f6c29da..f51fad9 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp16/config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp16/config.env @@ -35,7 +35,7 @@ NCCL_DEBUG="${NCCL_DEBUG:-WARN}" # max_running_requests must >= max concurrency tested (128) to avoid queueing # cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" -CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-256}" +CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-64}" MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}" # ---- Benchmark ----