[Feat] add 6000D DSV4 tiny adaptive benchmarks
This commit is contained in:
Quantong Qiu
parent
039a2fa7b9
commit
0c2256ca1c
@ -1,5 +1,9 @@
|
|||||||
# sskj — 多平台大模型推理性能基准测试项目
|
# sskj — 多平台大模型推理性能基准测试项目
|
||||||
|
|
||||||
|
> **更新（2026-07-21 11:54:30 +0800）**
|
||||||
|
> - 新增 RTX 6000D / DeepSeek-V4-Flash tiny 自适应并发实验：输出固定 1K，输入遍历 1K、2K、4K、8K、16K、32K、64K、128K，并发采用 `C=16` 起、每次加 16、首点失败时按 `16 -> 8 -> 1` 回退。
|
||||||
|
> - vLLM 覆盖 TP2/DP4、TP4/DP2、TP8/DP1，部署固定为 128K context、128 活跃请求、0.9 显存比例；SGLang 覆盖 TP4/DP2、TP8/DP1，固定为 128K、64、0.9，TP2/DP4 因 SM120 Marlin 权重加载 OOM 明确排除。
|
||||||
|
|
||||||
> **更新（2026-07-20 17:55:20 +0800）**
|
> **更新（2026-07-20 17:55:20 +0800）**
|
||||||
> - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`，使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph；Docker 与 native 启动入口均已覆盖。
|
> - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`，使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph；Docker 与 native 启动入口均已覆盖。
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,56 @@
|
|||||||
|
# Adaptive concurrency search settings.
|
||||||
|
#
|
||||||
|
# For each fixed (TP, DP, ISL, OSL), probe:
|
||||||
|
# C = start, start * multiplier, ... up to max
|
||||||
|
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
||||||
|
# PLATEAU_PATIENCE consecutive points.
|
||||||
|
|
||||||
|
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
||||||
|
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}"
|
||||||
|
|
||||||
|
# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
|
||||||
|
ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
|
||||||
|
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
||||||
|
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
||||||
|
|
||||||
|
# A gain below 2% is treated as throughput saturation. Two consecutive
|
||||||
|
# low-gain points prevent one noisy measurement from stopping the search.
|
||||||
|
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
||||||
|
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
||||||
|
|
||||||
|
# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
|
||||||
|
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
|
||||||
|
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
|
||||||
|
|
||||||
|
# Keep the same random workload semantics as the fixed matrix baseline.
|
||||||
|
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
||||||
|
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
||||||
|
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
||||||
|
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
||||||
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||||||
|
# Before each measured point, warm up with the same concurrency so lazy kernel
|
||||||
|
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
|
||||||
|
# cap; set a positive cap only when very high-concurrency warmup is impractical.
|
||||||
|
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
||||||
|
|
||||||
|
# Reject a point if the completed request count or actual token lengths do not
|
||||||
|
# match the requested workload.
|
||||||
|
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
||||||
|
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
||||||
|
|
||||||
|
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
||||||
|
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Optional space-separated filters, useful for smoke tests:
|
||||||
|
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
||||||
|
TP_LIST="${TP_LIST:-}"
|
||||||
|
ISL_LIST="${ISL_LIST:-}"
|
||||||
|
OSL_LIST="${OSL_LIST:-}"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D
|
||||||
|
# (8 GPUs) using SGLang. It fixes OSL=1024 and sweeps ISL=1K..128K.
|
||||||
|
# TP=2/DP=4 is excluded because Marlin weight loading OOMs before KV Cache
|
||||||
|
# initialization on this machine.
|
||||||
|
# Tests SGLang with two parallel configurations:
|
||||||
|
# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas
|
||||||
|
# TP=8, DP=1 -> 8 GPUs, no data parallelism
|
||||||
|
|
||||||
|
EXPERIMENT="dsv4_pro6000_sglang_tiny_1k_output"
|
||||||
|
MODEL_NAME="DeepSeek-V4-Flash"
|
||||||
|
MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
|
||||||
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
||||||
|
|
||||||
|
SGLANG_PORT="${SGLANG_PORT:-30031}"
|
||||||
|
|
||||||
|
# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
|
||||||
|
# and the benchmark client. Defaults to the system python3 if the sglang venv
|
||||||
|
# does not exist on the host.
|
||||||
|
VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
|
||||||
|
|
||||||
|
# Run the benchmark client natively (0) or inside Docker (1).
|
||||||
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||||
|
|
||||||
|
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||||||
|
# directory under this experiment so the benchmark is self-contained.
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
|
||||||
|
# Parallel configurations to test. Format: "TP DP"
|
||||||
|
declare -a PARALLEL_CONFIGS=(
|
||||||
|
"4 2"
|
||||||
|
"8 1"
|
||||||
|
)
|
||||||
|
|
||||||
|
# SGLang server settings, verified for this model and machine.
|
||||||
|
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}"
|
||||||
|
MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}"
|
||||||
|
CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}"
|
||||||
|
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
|
||||||
|
|
||||||
|
# Deployment switch. 0 = native sglang venv, 1 = Docker.
|
||||||
|
USE_DOCKER="${USE_DOCKER:-1}"
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
|
||||||
|
|
||||||
|
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||||
|
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/sglang_sm120_cache}"
|
||||||
|
|
||||||
|
# Matrix contains ISL=1K..128K with fixed OSL=1K.
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
|
||||||
|
# Sampling density for concurrency.
|
||||||
|
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
|
||||||
|
# 2 = only test the low and high endpoints.
|
||||||
|
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||||||
|
|
||||||
|
# Per-scenario timeout to avoid hangs (seconds).
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
|
||||||
|
# GPU memory sampling interval (seconds).
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Dry-run mode: if 1, only log the server args and scenario plan without starting
|
||||||
|
# any server or sending requests.
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
|
||||||
|
# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
|
|
||||||
|
# PyTorch CUDA allocator setting for the SGLang server. expandable_segments
|
||||||
|
# reduces fragmentation from the GiB-scale indexer temporaries that OOM the
|
||||||
|
# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096.
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
|
||||||
@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"comment": "Tiny adaptive test: fixed OSL=1024, ISL=1K..128K. Only Y entries are run.",
|
||||||
|
"mode": "Y",
|
||||||
|
"matrix": {
|
||||||
|
"1024": {"1024": "Y"},
|
||||||
|
"2048": {"1024": "Y"},
|
||||||
|
"4096": {"1024": "Y"},
|
||||||
|
"8192": {"1024": "Y"},
|
||||||
|
"16384": {"1024": "Y"},
|
||||||
|
"32768": {"1024": "Y"},
|
||||||
|
"65536": {"1024": "Y"},
|
||||||
|
"131072": {"1024": "Y"}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,193 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Tiny test: find Total-TPS saturation for fixed OSL=1K and ISL=1K..128K.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
|
||||||
|
ENGINE="sglang"
|
||||||
|
ENGINE_PORT="$SGLANG_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
|
||||||
|
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
|
||||||
|
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
|
||||||
|
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
elif kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
fi
|
||||||
|
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local -a args=(
|
||||||
|
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--tp-size "$tp"
|
||||||
|
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
||||||
|
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
||||||
|
--context-length "$CONTEXT_LENGTH"
|
||||||
|
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
)
|
||||||
|
if (( dp > 1 )); then
|
||||||
|
args+=(--dp-size "$dp")
|
||||||
|
fi
|
||||||
|
printf '%q ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting sglang server tp=${tp} dp=${dp}"
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
else
|
||||||
|
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
fi
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
fi
|
||||||
|
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend sglang
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
|
||||||
|
: # random-ids does not need --tokenize-prompt
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
--entrypoint python3 \
|
||||||
|
"$DOCKER_IMAGE" \
|
||||||
|
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
export SEARCH_START_CONCURRENCY=16
|
||||||
|
export SEARCH_ADDEND=16
|
||||||
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
|
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
|
||||||
|
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
||||||
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
105
experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh
Executable file
105
experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh
Executable file
@ -0,0 +1,105 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start SGLang server in Docker for a given TPxDP configuration.
|
||||||
|
# Usage: start_sglang_docker.sh <TP> <DP>
|
||||||
|
#
|
||||||
|
# Uses the verified SGLang SM120 image and keeps its JIT cache on
|
||||||
|
# persistent storage. The container is removed automatically on stop.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1}"
|
||||||
|
DP="${2}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
|
||||||
|
|
||||||
|
IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
|
||||||
|
PORT="${SGLANG_PORT:-30031}"
|
||||||
|
NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
|
||||||
|
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
|
||||||
|
|
||||||
|
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
|
||||||
|
# Clean up any stale container with the same name.
|
||||||
|
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
SERVER_ARGS=(
|
||||||
|
-m sglang.launch_server
|
||||||
|
--model-path "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--tp-size "$TP"
|
||||||
|
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
||||||
|
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
||||||
|
--context-length "$CONTEXT_LENGTH"
|
||||||
|
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$PORT"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$DP" -gt 1 ]]; then
|
||||||
|
SERVER_ARGS+=(
|
||||||
|
--dp-size "$DP"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||||
|
|
||||||
|
echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ==="
|
||||||
|
echo "Image: $IMAGE"
|
||||||
|
echo "Model: $MODEL_PATH"
|
||||||
|
echo "Container name: $NAME"
|
||||||
|
echo "Host port: $PORT"
|
||||||
|
echo "Command: python3 ${SERVER_ARGS_STR}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
|
||||||
|
# Run docker in the foreground so that killing the host process stops the
|
||||||
|
# container (the --rm flag ensures cleanup). nohup lets us background it and
|
||||||
|
# capture the host PID in the same way as the bare-metal start script.
|
||||||
|
nohup docker run --rm \
|
||||||
|
--name "$NAME" \
|
||||||
|
--gpus all \
|
||||||
|
--privileged \
|
||||||
|
--ipc=host \
|
||||||
|
--network host \
|
||||||
|
--ulimit memlock=-1 \
|
||||||
|
--ulimit stack=67108864 \
|
||||||
|
--entrypoint python3 \
|
||||||
|
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||||
|
-v "${CACHE_DIR}:/root/.cache" \
|
||||||
|
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
||||||
|
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
-e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \
|
||||||
|
"$IMAGE" \
|
||||||
|
"${SERVER_ARGS[@]}" \
|
||||||
|
> "$LOG" 2>&1 &
|
||||||
|
|
||||||
|
PID=$!
|
||||||
|
echo $PID > "$PID_FILE"
|
||||||
|
echo "PID: $PID"
|
||||||
|
echo "Waiting for health on port ${PORT}..."
|
||||||
|
|
||||||
|
for i in $(seq 1 240); do
|
||||||
|
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
||||||
|
echo "SGLang server is ready at http://127.0.0.1:${PORT}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 $PID 2>/dev/null; then
|
||||||
|
echo "ERROR: Docker SGLang server exited early"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting... ($i/240)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: Docker SGLang server not healthy after 240 retries"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
87
experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh
Executable file
87
experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh
Executable file
@ -0,0 +1,87 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start SGLang server for a given TP×DP configuration.
|
||||||
|
# Usage: start_sglang_dp.sh <TP> <DP>
|
||||||
|
#
|
||||||
|
# By default this delegates to the Docker start script because the experiment
|
||||||
|
# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the
|
||||||
|
# local VENV_CLIENT environment instead.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1}"
|
||||||
|
DP="${2}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
if [[ "${USE_DOCKER:-1}" == "1" ]]; then
|
||||||
|
exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@"
|
||||||
|
fi
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
|
VENV="${VENV_CLIENT}"
|
||||||
|
export PATH="$VENV/bin:$PATH"
|
||||||
|
export PYTHONUNBUFFERED=1
|
||||||
|
export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
|
||||||
|
export TMPDIR="${RUNTIME_BASE}/tmp"
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
|
||||||
|
|
||||||
|
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||||
|
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
|
||||||
|
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
|
||||||
|
SERVER_ARGS=(
|
||||||
|
python3 -m sglang.launch_server
|
||||||
|
--model-path "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--tp-size "$TP"
|
||||||
|
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
||||||
|
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
||||||
|
--context-length "$CONTEXT_LENGTH"
|
||||||
|
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$SGLANG_PORT"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$DP" -gt 1 ]]; then
|
||||||
|
SERVER_ARGS+=(
|
||||||
|
--dp-size "$DP"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||||
|
|
||||||
|
echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ==="
|
||||||
|
echo "Model: $MODEL_PATH"
|
||||||
|
echo "Port: $SGLANG_PORT"
|
||||||
|
echo "Command: $SERVER_ARGS_STR"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
|
||||||
|
nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
|
||||||
|
|
||||||
|
PID=$!
|
||||||
|
echo $PID > "$PID_FILE"
|
||||||
|
echo "PID: $PID"
|
||||||
|
echo "Waiting for health on port ${SGLANG_PORT}..."
|
||||||
|
|
||||||
|
for i in $(seq 1 240); do
|
||||||
|
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then
|
||||||
|
echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 $PID 2>/dev/null; then
|
||||||
|
echo "ERROR: SGLang server exited early"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting... ($i/240)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: SGLang server not healthy after 240 retries"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
# Adaptive concurrency search settings.
|
||||||
|
#
|
||||||
|
# For each fixed (TP, DP, ISL, OSL), probe:
|
||||||
|
# C = start, start * multiplier, ... up to max
|
||||||
|
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
||||||
|
# PLATEAU_PATIENCE consecutive points.
|
||||||
|
|
||||||
|
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
||||||
|
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}"
|
||||||
|
|
||||||
|
# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
|
||||||
|
ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
|
||||||
|
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
||||||
|
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
||||||
|
|
||||||
|
# A gain below 2% is treated as throughput saturation. Two consecutive
|
||||||
|
# low-gain points prevent one noisy measurement from stopping the search.
|
||||||
|
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
||||||
|
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
||||||
|
|
||||||
|
# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
|
||||||
|
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
|
||||||
|
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
|
||||||
|
|
||||||
|
# Keep the same random workload semantics as the fixed matrix baseline.
|
||||||
|
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
||||||
|
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
||||||
|
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
||||||
|
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
||||||
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||||||
|
# Before each measured point, warm up with the same concurrency so lazy kernel
|
||||||
|
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
|
||||||
|
# cap; set a positive cap only when very high-concurrency warmup is impractical.
|
||||||
|
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
||||||
|
|
||||||
|
# Reject a point if the completed request count or actual token lengths do not
|
||||||
|
# match the requested workload.
|
||||||
|
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
||||||
|
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
||||||
|
|
||||||
|
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
||||||
|
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Optional space-separated filters, useful for smoke tests:
|
||||||
|
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
||||||
|
TP_LIST="${TP_LIST:-}"
|
||||||
|
ISL_LIST="${ISL_LIST:-}"
|
||||||
|
OSL_LIST="${OSL_LIST:-}"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D
|
||||||
|
# (8 GPUs) using vLLM. It fixes OSL=1024 and sweeps ISL=1K..128K.
|
||||||
|
# Tests vLLM with three parallel configurations:
|
||||||
|
# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas
|
||||||
|
# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas
|
||||||
|
# TP=8, DP=1 -> 8 GPUs, no data parallelism
|
||||||
|
|
||||||
|
EXPERIMENT="dsv4_pro6000_vllm_tiny_1k_output"
|
||||||
|
MODEL_NAME="DeepSeek-V4-Flash"
|
||||||
|
MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
|
||||||
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
||||||
|
|
||||||
|
VLLM_PORT="${VLLM_PORT:-30030}"
|
||||||
|
|
||||||
|
# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
|
||||||
|
# and the benchmark client. Defaults to the system python3 if the sglang venv
|
||||||
|
# does not exist on the host.
|
||||||
|
VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
|
||||||
|
|
||||||
|
# Run the benchmark client natively (0) or inside Docker (1).
|
||||||
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||||
|
|
||||||
|
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||||||
|
# directory under this experiment so the benchmark is self-contained.
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
|
||||||
|
# Parallel configurations to test. Format: "TP DP"
|
||||||
|
declare -a PARALLEL_CONFIGS=(
|
||||||
|
"2 4"
|
||||||
|
"4 2"
|
||||||
|
"8 1"
|
||||||
|
)
|
||||||
|
|
||||||
|
# vLLM server settings, verified for this model and machine.
|
||||||
|
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
|
||||||
|
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
|
||||||
|
BLOCK_SIZE="${BLOCK_SIZE:-256}"
|
||||||
|
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||||||
|
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
|
||||||
|
|
||||||
|
# Deployment switch. 0 = native vllm venv, 1 = Docker.
|
||||||
|
USE_DOCKER="${USE_DOCKER:-1}"
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
|
||||||
|
|
||||||
|
# Benchmark client Docker image. vLLM's image does not include the SGLang benchmark client,
|
||||||
|
# so the client runs inside the SGLang image and targets the vLLM backend via the
|
||||||
|
# OpenAI-compatible API.
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
|
||||||
|
|
||||||
|
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||||
|
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/vllm_sm120_cache}"
|
||||||
|
|
||||||
|
# Matrix contains ISL=1K..128K with fixed OSL=1K.
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
|
||||||
|
# Sampling density for concurrency.
|
||||||
|
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
|
||||||
|
# 2 = only test the low and high endpoints.
|
||||||
|
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||||||
|
|
||||||
|
# Per-scenario timeout to avoid hangs (seconds).
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
|
||||||
|
# GPU memory sampling interval (seconds).
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Dry-run mode: if 1, only log the server args and scenario plan without starting
|
||||||
|
# any server or sending requests.
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
|
||||||
|
# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"comment": "Tiny adaptive test: fixed OSL=1024, ISL=1K..128K. Only Y entries are run.",
|
||||||
|
"mode": "Y",
|
||||||
|
"matrix": {
|
||||||
|
"1024": {"1024": "Y"},
|
||||||
|
"2048": {"1024": "Y"},
|
||||||
|
"4096": {"1024": "Y"},
|
||||||
|
"8192": {"1024": "Y"},
|
||||||
|
"16384": {"1024": "Y"},
|
||||||
|
"32768": {"1024": "Y"},
|
||||||
|
"65536": {"1024": "Y"},
|
||||||
|
"131072": {"1024": "Y"}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,175 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Tiny test: find Total-TPS saturation for fixed OSL=1K and ISL=1K..128K.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
|
||||||
|
ENGINE="vllm"
|
||||||
|
ENGINE_PORT="$VLLM_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
|
||||||
|
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||||
|
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
fi
|
||||||
|
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local -a args=(
|
||||||
|
vllm serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$tp"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--max-num-seqs "$MAX_NUM_SEQS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
)
|
||||||
|
if (( dp > 1 )); then
|
||||||
|
args+=(--data-parallel-size "$dp")
|
||||||
|
fi
|
||||||
|
printf '%q ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting vllm server tp=${tp} dp=${dp}"
|
||||||
|
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend vllm
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
|
||||||
|
: # random-ids does not need --tokenize-prompt
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
--entrypoint python3 \
|
||||||
|
"$DOCKER_CLIENT_IMAGE" \
|
||||||
|
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
export SEARCH_START_CONCURRENCY=16
|
||||||
|
export SEARCH_ADDEND=16
|
||||||
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
|
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
|
||||||
|
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
||||||
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
114
experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh
Executable file
114
experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh
Executable file
@ -0,0 +1,114 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start vLLM server in Docker for a given TPxDP configuration.
|
||||||
|
# Usage: start_vllm_docker.sh <TP> <DP>
|
||||||
|
#
|
||||||
|
# Uses the verified vLLM SM120 image and keeps its cache on persistent
|
||||||
|
# storage. The container is removed automatically on stop.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1}"
|
||||||
|
DP="${2}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
|
||||||
|
|
||||||
|
IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
|
||||||
|
PORT="${VLLM_PORT:-30030}"
|
||||||
|
NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
|
||||||
|
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
||||||
|
|
||||||
|
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
|
||||||
|
# Clean up any stale container with the same name.
|
||||||
|
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
SERVER_ARGS=(
|
||||||
|
serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$TP"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--max-num-seqs "$MAX_NUM_SEQS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$PORT"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$DP" -gt 1 ]]; then
|
||||||
|
SERVER_ARGS+=(
|
||||||
|
--data-parallel-size "$DP"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||||
|
|
||||||
|
echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ==="
|
||||||
|
echo "Image: $IMAGE"
|
||||||
|
echo "Model: $MODEL_PATH"
|
||||||
|
echo "Container name: $NAME"
|
||||||
|
echo "Host port: $PORT"
|
||||||
|
echo "Command: vllm ${SERVER_ARGS_STR}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
|
||||||
|
# Run docker in the foreground so that killing the host process stops the
|
||||||
|
# container (the --rm flag ensures cleanup). nohup lets us background it and
|
||||||
|
# capture the host PID in the same way as the bare-metal start script.
|
||||||
|
nohup docker run --rm \
|
||||||
|
--name "$NAME" \
|
||||||
|
--gpus all \
|
||||||
|
--privileged \
|
||||||
|
--ipc=host \
|
||||||
|
--network host \
|
||||||
|
--ulimit memlock=-1 \
|
||||||
|
--ulimit stack=67108864 \
|
||||||
|
--entrypoint vllm \
|
||||||
|
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||||
|
-v "${CACHE_DIR}:/root/.cache" \
|
||||||
|
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
||||||
|
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
"$IMAGE" \
|
||||||
|
"${SERVER_ARGS[@]}" \
|
||||||
|
> "$LOG" 2>&1 &
|
||||||
|
|
||||||
|
PID=$!
|
||||||
|
echo $PID > "$PID_FILE"
|
||||||
|
echo "PID: $PID"
|
||||||
|
echo "Waiting for health on port ${PORT}..."
|
||||||
|
|
||||||
|
for i in $(seq 1 600); do
|
||||||
|
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
||||||
|
echo "vLLM server is ready at http://127.0.0.1:${PORT}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
# Check if the Docker container is still running (not the nohup PID).
|
||||||
|
# Allow a brief grace period for the container to appear in docker ps.
|
||||||
|
container_running=0
|
||||||
|
for _ in $(seq 1 3); do
|
||||||
|
if docker ps --filter "name=${NAME}" --format '{{.Names}}' | grep -q "^${NAME}$"; then
|
||||||
|
container_running=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if [[ "$container_running" -eq 0 ]]; then
|
||||||
|
echo "ERROR: Docker vLLM container exited early"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting... ($i/600)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: Docker vLLM server not healthy after 600 retries"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
86
experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh
Executable file
86
experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh
Executable file
@ -0,0 +1,86 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start vLLM server for a given TP×DP configuration.
|
||||||
|
# Usage: start_vllm_dp.sh <TP> <DP>
|
||||||
|
#
|
||||||
|
# By default this delegates to the Docker start script because the experiment
|
||||||
|
# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the
|
||||||
|
# local VENV_VLLM environment instead.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1}"
|
||||||
|
DP="${2}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
if [[ "${USE_DOCKER:-1}" == "1" ]]; then
|
||||||
|
exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@"
|
||||||
|
fi
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
|
VENV="${VENV_VLLM}"
|
||||||
|
export PATH="$VENV/bin:$PATH"
|
||||||
|
export PYTHONUNBUFFERED=1
|
||||||
|
export TMPDIR="${RUNTIME_BASE}/tmp"
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
|
||||||
|
|
||||||
|
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||||
|
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
||||||
|
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
|
||||||
|
SERVER_ARGS=(
|
||||||
|
vllm serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$TP"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--max-num-seqs "$MAX_NUM_SEQS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$VLLM_PORT"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$DP" -gt 1 ]]; then
|
||||||
|
SERVER_ARGS+=(
|
||||||
|
--data-parallel-size "$DP"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||||
|
|
||||||
|
echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ==="
|
||||||
|
echo "Model: $MODEL_PATH"
|
||||||
|
echo "Port: $VLLM_PORT"
|
||||||
|
echo "Command: $SERVER_ARGS_STR"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
|
||||||
|
nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
|
||||||
|
|
||||||
|
PID=$!
|
||||||
|
echo $PID > "$PID_FILE"
|
||||||
|
echo "PID: $PID"
|
||||||
|
echo "Waiting for health on port ${VLLM_PORT}..."
|
||||||
|
|
||||||
|
for i in $(seq 1 600); do
|
||||||
|
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
|
||||||
|
echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 $PID 2>/dev/null; then
|
||||||
|
echo "ERROR: vLLM server exited early"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting... ($i/600)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: vLLM server not healthy after 600 retries"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
Loading…
x
Reference in New Issue
Block a user