add run_adaptive_concurrency_add16
This commit is contained in:
iiGray
parent
a7f453b997
commit
60826dda09
184
experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
184
experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env bash
|
||||
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
|
||||
|
||||
ENGINE="sglang"
|
||||
ENGINE_PORT="$SGLANG_PORT"
|
||||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
|
||||
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||
PYTHON="${VENV_CLIENT}/bin/python"
|
||||
else
|
||||
PYTHON="$(command -v python3)"
|
||||
fi
|
||||
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
|
||||
|
||||
engine_is_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 \
|
||||
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
engine_stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
|
||||
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
pid="$(cat "$pid_file")"
|
||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
|
||||
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
elif kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
fi
|
||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
sleep 2
|
||||
}
|
||||
|
||||
engine_build_server_args() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local -a args=(
|
||||
sglang serve --model-path "$MODEL_PATH"
|
||||
--trust-remote-code
|
||||
--tp-size "$tp"
|
||||
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
||||
--context-length "$CONTEXT_LENGTH"
|
||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
||||
--host 0.0.0.0
|
||||
--port "$ENGINE_PORT"
|
||||
)
|
||||
if (( dp > 1 )); then
|
||||
args+=(--dp-size "$dp")
|
||||
fi
|
||||
printf '%q ' "${args[@]}"
|
||||
}
|
||||
|
||||
engine_start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||
log "starting sglang server tp=${tp} dp=${dp}"
|
||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
else
|
||||
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
fi
|
||||
if ! engine_is_healthy; then
|
||||
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
|
||||
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||
)"
|
||||
fi
|
||||
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
|
||||
}
|
||||
|
||||
engine_detect_oom() {
|
||||
local detail_log="$1"
|
||||
local tp="$2"
|
||||
local dp="$3"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||
local -a logs=("$detail_log" "$outer_log")
|
||||
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||
fi
|
||||
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
|
||||
return $?
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
engine_run_bench() {
|
||||
local isl="$1"
|
||||
local dsl="$2"
|
||||
local concurrency="$3"
|
||||
local num_prompts="$4"
|
||||
local output_file="$5"
|
||||
local warmup_requests
|
||||
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||
local -a bench_args=(
|
||||
--backend sglang
|
||||
--host 127.0.0.1
|
||||
--port "$ENGINE_PORT"
|
||||
--dataset-name "$BENCH_DATASET_NAME"
|
||||
--random-input-len "$isl"
|
||||
--random-output-len "$dsl"
|
||||
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--request-rate 10000
|
||||
--warmup-requests "$warmup_requests"
|
||||
--output-file "$output_file"
|
||||
--output-details
|
||||
--disable-tqdm
|
||||
)
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||
else
|
||||
bench_args+=(--tokenize-prompt)
|
||||
fi
|
||||
|
||||
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||
fi
|
||||
docker run --rm \
|
||||
--network host \
|
||||
"${volume_args[@]}" \
|
||||
-e PYTHONUNBUFFERED=1 \
|
||||
"$DOCKER_IMAGE" \
|
||||
python -m sglang.bench_serving "${bench_args[@]}"
|
||||
else
|
||||
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
export -f engine_run_bench
|
||||
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
||||
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||
|
||||
export SEARCH_START_CONCURRENCY=16
|
||||
export SEARCH_ADDEND=16
|
||||
|
||||
adaptive_main "$@"
|
||||
167
experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
167
experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/DSL shape.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
|
||||
|
||||
ENGINE="vllm"
|
||||
ENGINE_PORT="$VLLM_PORT"
|
||||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
|
||||
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||
PYTHON="${VENV_CLIENT}/bin/python"
|
||||
else
|
||||
PYTHON="$(command -v python3)"
|
||||
fi
|
||||
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||
|
||||
engine_is_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 \
|
||||
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
engine_stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
pid="$(cat "$pid_file")"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
fi
|
||||
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
sleep 2
|
||||
}
|
||||
|
||||
engine_build_server_args() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local -a args=(
|
||||
vllm serve "$MODEL_PATH"
|
||||
--trust-remote-code
|
||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||
--block-size "$BLOCK_SIZE"
|
||||
--tensor-parallel-size "$tp"
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||
--max-model-len "$MAX_MODEL_LEN"
|
||||
--max-num-seqs "$MAX_NUM_SEQS"
|
||||
--no-enable-flashinfer-autotune
|
||||
--host 0.0.0.0
|
||||
--port "$ENGINE_PORT"
|
||||
)
|
||||
if (( dp > 1 )); then
|
||||
args+=(--data-parallel-size "$dp")
|
||||
fi
|
||||
printf '%q ' "${args[@]}"
|
||||
}
|
||||
|
||||
engine_start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||
log "starting vllm server tp=${tp} dp=${dp}"
|
||||
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
if ! engine_is_healthy; then
|
||||
log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||
-name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
|
||||
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||
)"
|
||||
log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
|
||||
}
|
||||
|
||||
engine_detect_oom() {
|
||||
local detail_log="$1"
|
||||
local tp="$2"
|
||||
local dp="$3"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||
local -a logs=("$detail_log" "$outer_log")
|
||||
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||
fi
|
||||
grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
|
||||
}
|
||||
|
||||
engine_run_bench() {
|
||||
local isl="$1"
|
||||
local dsl="$2"
|
||||
local concurrency="$3"
|
||||
local num_prompts="$4"
|
||||
local output_file="$5"
|
||||
local warmup_requests
|
||||
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||
local -a bench_args=(
|
||||
--backend vllm
|
||||
--host 127.0.0.1
|
||||
--port "$ENGINE_PORT"
|
||||
--dataset-name "$BENCH_DATASET_NAME"
|
||||
--random-input-len "$isl"
|
||||
--random-output-len "$dsl"
|
||||
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--request-rate 10000
|
||||
--warmup-requests "$warmup_requests"
|
||||
--output-file "$output_file"
|
||||
--output-details
|
||||
--disable-tqdm
|
||||
)
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||
else
|
||||
bench_args+=(--tokenize-prompt)
|
||||
fi
|
||||
|
||||
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||
fi
|
||||
docker run --rm \
|
||||
--network host \
|
||||
"${volume_args[@]}" \
|
||||
-e PYTHONUNBUFFERED=1 \
|
||||
-e HF_HUB_OFFLINE=1 \
|
||||
-e TRANSFORMERS_OFFLINE=1 \
|
||||
"$DOCKER_CLIENT_IMAGE" \
|
||||
python -m sglang.bench_serving "${bench_args[@]}"
|
||||
else
|
||||
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
export -f engine_run_bench
|
||||
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||
|
||||
export SEARCH_START_CONCURRENCY=16
|
||||
export SEARCH_ADDEND=16
|
||||
|
||||
adaptive_main "$@"
|
||||
184
experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
184
experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env bash
|
||||
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
|
||||
|
||||
ENGINE="sglang"
|
||||
ENGINE_PORT="$SGLANG_PORT"
|
||||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
|
||||
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||
PYTHON="${VENV_CLIENT}/bin/python"
|
||||
else
|
||||
PYTHON="$(command -v python3)"
|
||||
fi
|
||||
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
|
||||
|
||||
engine_is_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 \
|
||||
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
engine_stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
|
||||
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
pid="$(cat "$pid_file")"
|
||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
|
||||
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
elif kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
fi
|
||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
sleep 2
|
||||
}
|
||||
|
||||
engine_build_server_args() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local -a args=(
|
||||
sglang serve --model-path "$MODEL_PATH"
|
||||
--trust-remote-code
|
||||
--tp-size "$tp"
|
||||
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
||||
--context-length "$CONTEXT_LENGTH"
|
||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
||||
--host 0.0.0.0
|
||||
--port "$ENGINE_PORT"
|
||||
)
|
||||
if (( dp > 1 )); then
|
||||
args+=(--dp-size "$dp")
|
||||
fi
|
||||
printf '%q ' "${args[@]}"
|
||||
}
|
||||
|
||||
engine_start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||
log "starting sglang server tp=${tp} dp=${dp}"
|
||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
else
|
||||
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
fi
|
||||
if ! engine_is_healthy; then
|
||||
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
|
||||
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||
)"
|
||||
fi
|
||||
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
|
||||
}
|
||||
|
||||
engine_detect_oom() {
|
||||
local detail_log="$1"
|
||||
local tp="$2"
|
||||
local dp="$3"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||
local -a logs=("$detail_log" "$outer_log")
|
||||
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||
fi
|
||||
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
|
||||
return $?
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
engine_run_bench() {
|
||||
local isl="$1"
|
||||
local osl="$2"
|
||||
local concurrency="$3"
|
||||
local num_prompts="$4"
|
||||
local output_file="$5"
|
||||
local warmup_requests
|
||||
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||
local -a bench_args=(
|
||||
--backend sglang
|
||||
--host 127.0.0.1
|
||||
--port "$ENGINE_PORT"
|
||||
--dataset-name "$BENCH_DATASET_NAME"
|
||||
--random-input-len "$isl"
|
||||
--random-output-len "$osl"
|
||||
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--request-rate 10000
|
||||
--warmup-requests "$warmup_requests"
|
||||
--output-file "$output_file"
|
||||
--output-details
|
||||
--disable-tqdm
|
||||
)
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||
else
|
||||
bench_args+=(--tokenize-prompt)
|
||||
fi
|
||||
|
||||
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||
fi
|
||||
docker run --rm \
|
||||
--network host \
|
||||
"${volume_args[@]}" \
|
||||
-e PYTHONUNBUFFERED=1 \
|
||||
"$DOCKER_IMAGE" \
|
||||
python -m sglang.bench_serving "${bench_args[@]}"
|
||||
else
|
||||
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
export -f engine_run_bench
|
||||
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
||||
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||
|
||||
export SEARCH_START_CONCURRENCY=16
|
||||
export SEARCH_ADDEND=16
|
||||
|
||||
adaptive_main "$@"
|
||||
167
experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
167
experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
|
||||
|
||||
ENGINE="vllm"
|
||||
ENGINE_PORT="$VLLM_PORT"
|
||||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
|
||||
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||
PYTHON="${VENV_CLIENT}/bin/python"
|
||||
else
|
||||
PYTHON="$(command -v python3)"
|
||||
fi
|
||||
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||
|
||||
engine_is_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 \
|
||||
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
engine_stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
pid="$(cat "$pid_file")"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
fi
|
||||
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
sleep 2
|
||||
}
|
||||
|
||||
engine_build_server_args() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local -a args=(
|
||||
vllm serve "$MODEL_PATH"
|
||||
--trust-remote-code
|
||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||
--block-size "$BLOCK_SIZE"
|
||||
--tensor-parallel-size "$tp"
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||
--max-model-len "$MAX_MODEL_LEN"
|
||||
--max-num-seqs "$MAX_NUM_SEQS"
|
||||
--no-enable-flashinfer-autotune
|
||||
--host 0.0.0.0
|
||||
--port "$ENGINE_PORT"
|
||||
)
|
||||
if (( dp > 1 )); then
|
||||
args+=(--data-parallel-size "$dp")
|
||||
fi
|
||||
printf '%q ' "${args[@]}"
|
||||
}
|
||||
|
||||
engine_start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||
log "starting vllm server tp=${tp} dp=${dp}"
|
||||
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
if ! engine_is_healthy; then
|
||||
log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||
-name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
|
||||
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||
)"
|
||||
log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
|
||||
}
|
||||
|
||||
engine_detect_oom() {
|
||||
local detail_log="$1"
|
||||
local tp="$2"
|
||||
local dp="$3"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||
local -a logs=("$detail_log" "$outer_log")
|
||||
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||
fi
|
||||
grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
|
||||
}
|
||||
|
||||
engine_run_bench() {
|
||||
local isl="$1"
|
||||
local osl="$2"
|
||||
local concurrency="$3"
|
||||
local num_prompts="$4"
|
||||
local output_file="$5"
|
||||
local warmup_requests
|
||||
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||
local -a bench_args=(
|
||||
--backend vllm
|
||||
--host 127.0.0.1
|
||||
--port "$ENGINE_PORT"
|
||||
--dataset-name "$BENCH_DATASET_NAME"
|
||||
--random-input-len "$isl"
|
||||
--random-output-len "$osl"
|
||||
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--request-rate 10000
|
||||
--warmup-requests "$warmup_requests"
|
||||
--output-file "$output_file"
|
||||
--output-details
|
||||
--disable-tqdm
|
||||
)
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||
else
|
||||
bench_args+=(--tokenize-prompt)
|
||||
fi
|
||||
|
||||
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||
fi
|
||||
docker run --rm \
|
||||
--network host \
|
||||
"${volume_args[@]}" \
|
||||
-e PYTHONUNBUFFERED=1 \
|
||||
-e HF_HUB_OFFLINE=1 \
|
||||
-e TRANSFORMERS_OFFLINE=1 \
|
||||
"$DOCKER_CLIENT_IMAGE" \
|
||||
python -m sglang.bench_serving "${bench_args[@]}"
|
||||
else
|
||||
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
export -f engine_run_bench
|
||||
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||
|
||||
export SEARCH_START_CONCURRENCY=16
|
||||
export SEARCH_ADDEND=16
|
||||
|
||||
adaptive_main "$@"
|
||||
@ -29,7 +29,7 @@ export -f adaptive_warmup_request_count
|
||||
|
||||
adaptive_validate_config() {
|
||||
local name
|
||||
for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY SEARCH_MULTIPLIER \
|
||||
for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY \
|
||||
NUM_PROMPTS_MULTIPLIER PLATEAU_PATIENCE MAX_POINT_RETRIES \
|
||||
BENCH_WARMUP_MAX_REQUESTS; do
|
||||
if ! [[ "${!name}" =~ ^[0-9]+$ ]]; then
|
||||
@ -45,10 +45,17 @@ adaptive_validate_config() {
|
||||
log "ERROR: SEARCH_MAX_CONCURRENCY must be >= SEARCH_START_CONCURRENCY"
|
||||
return 1
|
||||
fi
|
||||
if (( SEARCH_MULTIPLIER < 2 )); then
|
||||
if [[ -n "${SEARCH_ADDEND:-}" ]]; then
|
||||
if ! [[ "${SEARCH_ADDEND}" =~ ^[0-9]+$ ]] || (( SEARCH_ADDEND < 1 )); then
|
||||
log "ERROR: SEARCH_ADDEND must be at least 1"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
if ! [[ "${SEARCH_MULTIPLIER:-}" =~ ^[0-9]+$ ]] || (( SEARCH_MULTIPLIER < 2 )); then
|
||||
log "ERROR: SEARCH_MULTIPLIER must be at least 2"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if (( NUM_PROMPTS_MULTIPLIER < 1 || PLATEAU_PATIENCE < 1 )); then
|
||||
log "ERROR: NUM_PROMPTS_MULTIPLIER and PLATEAU_PATIENCE must be at least 1"
|
||||
return 1
|
||||
@ -444,7 +451,12 @@ adaptive_run_shape() {
|
||||
stop_reason="SEARCH_CAP_REACHED"
|
||||
break
|
||||
fi
|
||||
local next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
|
||||
local next_concurrency
|
||||
if [[ -n "${SEARCH_ADDEND:-}" ]]; then
|
||||
next_concurrency=$((concurrency + SEARCH_ADDEND))
|
||||
else
|
||||
next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
|
||||
fi
|
||||
if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then
|
||||
next_concurrency="$SEARCH_MAX_CONCURRENCY"
|
||||
fi
|
||||
@ -473,7 +485,12 @@ adaptive_print_sequence() {
|
||||
if (( concurrency == SEARCH_MAX_CONCURRENCY )); then
|
||||
break
|
||||
fi
|
||||
local next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
|
||||
local next_concurrency
|
||||
if [[ -n "${SEARCH_ADDEND:-}" ]]; then
|
||||
next_concurrency=$((concurrency + SEARCH_ADDEND))
|
||||
else
|
||||
next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
|
||||
fi
|
||||
if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then
|
||||
next_concurrency="$SEARCH_MAX_CONCURRENCY"
|
||||
fi
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user