add run_adaptive_concurrency_add16

experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="sglang"
+ENGINE_PORT="$SGLANG_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if [[ -n "${CONTAINER_NAME:-}" ]]; then
+      if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
+        log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
+        docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
+        sleep 5
+        docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
+      fi
+    elif kill -0 "$pid" 2>/dev/null; then
+      log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  if [[ -z "${CONTAINER_NAME:-}" ]]; then
+    docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  fi
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    sglang serve --model-path "$MODEL_PATH"
+    --trust-remote-code
+    --tp-size "$tp"
+    --moe-runner-backend "$MOE_RUNNER_BACKEND"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--dp-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
+  log "starting sglang server tp=${tp} dp=${dp}"
+  if [[ -n "${CONTAINER_NAME:-}" ]]; then
+    bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  else
+    bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  fi
+  if ! engine_is_healthy; then
+    log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  if [[ -z "${CONTAINER_NAME:-}" ]]; then
+    ACTIVE_ENGINE_SERVER_LOG="$(
+      find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+        -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
+        -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+    )"
+  fi
+  log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
+    return 0
+  fi
+  if [[ -n "${CONTAINER_NAME:-}" ]]; then
+    docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
+    return $?
+  fi
+  return 1
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local dsl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend sglang
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$dsl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      "$DOCKER_IMAGE" \
+      python -m sglang.bench_serving "${bench_args[@]}"
+  else
+    "$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
+
+export SEARCH_START_CONCURRENCY=16
+export SEARCH_ADDEND=16
+
+adaptive_main "$@"

experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/DSL shape.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="vllm"
+ENGINE_PORT="$VLLM_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if kill -0 "$pid" 2>/dev/null; then
+      log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    vllm serve "$MODEL_PATH"
+    --trust-remote-code
+    --kv-cache-dtype "$KV_CACHE_DTYPE"
+    --block-size "$BLOCK_SIZE"
+    --tensor-parallel-size "$tp"
+    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --no-enable-flashinfer-autotune
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--data-parallel-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  log "starting vllm server tp=${tp} dp=${dp}"
+  bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  if ! engine_is_healthy; then
+    log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  ACTIVE_ENGINE_SERVER_LOG="$(
+    find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+      -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
+      -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+  )"
+  log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local dsl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend vllm
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$dsl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      -e HF_HUB_OFFLINE=1 \
+      -e TRANSFORMERS_OFFLINE=1 \
+      "$DOCKER_CLIENT_IMAGE" \
+      python -m sglang.bench_serving "${bench_args[@]}"
+  else
+    "$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
+
+export SEARCH_START_CONCURRENCY=16
+export SEARCH_ADDEND=16
+
+adaptive_main "$@"

experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="sglang"
+ENGINE_PORT="$SGLANG_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if [[ -n "${CONTAINER_NAME:-}" ]]; then
+      if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
+        log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
+        docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
+        sleep 5
+        docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
+      fi
+    elif kill -0 "$pid" 2>/dev/null; then
+      log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  if [[ -z "${CONTAINER_NAME:-}" ]]; then
+    docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  fi
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    sglang serve --model-path "$MODEL_PATH"
+    --trust-remote-code
+    --tp-size "$tp"
+    --moe-runner-backend "$MOE_RUNNER_BACKEND"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--dp-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
+  log "starting sglang server tp=${tp} dp=${dp}"
+  if [[ -n "${CONTAINER_NAME:-}" ]]; then
+    bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  else
+    bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  fi
+  if ! engine_is_healthy; then
+    log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  if [[ -z "${CONTAINER_NAME:-}" ]]; then
+    ACTIVE_ENGINE_SERVER_LOG="$(
+      find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+        -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
+        -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+    )"
+  fi
+  log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
+    return 0
+  fi
+  if [[ -n "${CONTAINER_NAME:-}" ]]; then
+    docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
+    return $?
+  fi
+  return 1
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local osl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend sglang
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$osl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      "$DOCKER_IMAGE" \
+      python -m sglang.bench_serving "${bench_args[@]}"
+  else
+    "$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
+
+export SEARCH_START_CONCURRENCY=16
+export SEARCH_ADDEND=16
+
+adaptive_main "$@"

experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="vllm"
+ENGINE_PORT="$VLLM_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if kill -0 "$pid" 2>/dev/null; then
+      log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    vllm serve "$MODEL_PATH"
+    --trust-remote-code
+    --kv-cache-dtype "$KV_CACHE_DTYPE"
+    --block-size "$BLOCK_SIZE"
+    --tensor-parallel-size "$tp"
+    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --no-enable-flashinfer-autotune
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--data-parallel-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  log "starting vllm server tp=${tp} dp=${dp}"
+  bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  if ! engine_is_healthy; then
+    log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  ACTIVE_ENGINE_SERVER_LOG="$(
+    find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+      -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
+      -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+  )"
+  log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local osl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend vllm
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$osl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      -e HF_HUB_OFFLINE=1 \
+      -e TRANSFORMERS_OFFLINE=1 \
+      "$DOCKER_CLIENT_IMAGE" \
+      python -m sglang.bench_serving "${bench_args[@]}"
+  else
+    "$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
+
+export SEARCH_START_CONCURRENCY=16
+export SEARCH_ADDEND=16
+
+adaptive_main "$@"

scripts/common/adaptive_bench_lib.sh

@@ -29,7 +29,7 @@ export -f adaptive_warmup_request_count
 
 adaptive_validate_config() {
   local name
-  for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY SEARCH_MULTIPLIER \
+  for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY \
     NUM_PROMPTS_MULTIPLIER PLATEAU_PATIENCE MAX_POINT_RETRIES \
     BENCH_WARMUP_MAX_REQUESTS; do
     if ! [[ "${!name}" =~ ^[0-9]+$ ]]; then
@@ -45,9 +45,16 @@ adaptive_validate_config() {
     log "ERROR: SEARCH_MAX_CONCURRENCY must be >= SEARCH_START_CONCURRENCY"
     return 1
   fi
-  if (( SEARCH_MULTIPLIER < 2 )); then
+  if [[ -n "${SEARCH_ADDEND:-}" ]]; then
-    log "ERROR: SEARCH_MULTIPLIER must be at least 2"
+    if ! [[ "${SEARCH_ADDEND}" =~ ^[0-9]+$ ]] || (( SEARCH_ADDEND < 1 )); then
-    return 1
+      log "ERROR: SEARCH_ADDEND must be at least 1"
+      return 1
+    fi
+  else
+    if ! [[ "${SEARCH_MULTIPLIER:-}" =~ ^[0-9]+$ ]] || (( SEARCH_MULTIPLIER < 2 )); then
+      log "ERROR: SEARCH_MULTIPLIER must be at least 2"
+      return 1
+    fi
   fi
   if (( NUM_PROMPTS_MULTIPLIER < 1 || PLATEAU_PATIENCE < 1 )); then
     log "ERROR: NUM_PROMPTS_MULTIPLIER and PLATEAU_PATIENCE must be at least 1"
@@ -444,7 +451,12 @@ adaptive_run_shape() {
       stop_reason="SEARCH_CAP_REACHED"
       break
     fi
-    local next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
+    local next_concurrency
+    if [[ -n "${SEARCH_ADDEND:-}" ]]; then
+      next_concurrency=$((concurrency + SEARCH_ADDEND))
+    else
+      next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
+    fi
     if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then
       next_concurrency="$SEARCH_MAX_CONCURRENCY"
     fi
@@ -473,7 +485,12 @@ adaptive_print_sequence() {
     if (( concurrency == SEARCH_MAX_CONCURRENCY )); then
       break
     fi
-    local next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
+    local next_concurrency
+    if [[ -n "${SEARCH_ADDEND:-}" ]]; then
+      next_concurrency=$((concurrency + SEARCH_ADDEND))
+    else
+      next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
+    fi
     if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then
       next_concurrency="$SEARCH_MAX_CONCURRENCY"
     fi