qqtang/sskj
1
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: max_running_requests=256, cuda_graph_max_bs=256 to match max concurrency 128

Browse Source
This commit is contained in:
Zhiyi Hong 2026-07-28 17:20:25 +08:00
parent da1d4d3d78
commit cd362c5ca0
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
experiments/pro6000/dsv4_pro6000_sglang_tp16/config.env
View File

@ -32,9 +32,11 @@ NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth0}"
NCCL_DEBUG="${NCCL_DEBUG:-WARN}" NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
# ---- SGLang memory / perf ---- # ---- SGLang memory / perf ----
# max_running_requests must >= max concurrency tested (128) to avoid queueing
# cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-8}" CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-256}"
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-32}" MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}"
# ---- Benchmark ---- # ---- Benchmark ----
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
@ -44,10 +46,6 @@ RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
WARMUP_REQUESTS="${WARMUP_REQUESTS:-16}" WARMUP_REQUESTS="${WARMUP_REQUESTS:-16}"
NUM_REPETITIONS="${NUM_REPETITIONS:-3}" NUM_REPETITIONS="${NUM_REPETITIONS:-3}"
# ---- SLO ----
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
# ---- Timeouts ---- # ---- Timeouts ----
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}" HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}"