diff --git a/README.md b/README.md index 19e303b..cc4bebe 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # sskj — 多平台大模型推理性能基准测试项目 +> **更新(2026-07-21 11:54:30 +0800)** +> - 新增 RTX 6000D / DeepSeek-V4-Flash tiny 自适应并发实验:输出固定 1K,输入遍历 1K、2K、4K、8K、16K、32K、64K、128K,并发采用 `C=16` 起、每次加 16、首点失败时按 `16 -> 8 -> 1` 回退。 +> - vLLM 覆盖 TP2/DP4、TP4/DP2、TP8/DP1,部署固定为 128K context、128 活跃请求、0.9 显存比例;SGLang 覆盖 TP4/DP2、TP8/DP1,固定为 128K、64、0.9,TP2/DP4 因 SM120 Marlin 权重加载 OOM 明确排除。 + > **更新(2026-07-20 17:55:20 +0800)** > - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`,使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph;Docker 与 native 启动入口均已覆盖。 diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/adaptive_config.env new file mode 100644 index 0000000..3cf7dd7 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/adaptive_config.env @@ -0,0 +1,56 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/config.env new file mode 100644 index 0000000..0eed7e1 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/config.env @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D +# (8 GPUs) using SGLang. It fixes OSL=1024 and sweeps ISL=1K..128K. +# TP=2/DP=4 is excluded because Marlin weight loading OOMs before KV Cache +# initialization on this machine. +# Tests SGLang with two parallel configurations: +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_pro6000_sglang_tiny_1k_output" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/6000D/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +SGLANG_PORT="${SGLANG_PORT:-30031}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "4 2" + "8 1" +) + +# SGLang server settings, verified for this model and machine. +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}" +MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}" +CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" + +# Deployment switch. 0 = native sglang venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" + +# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}" +SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}" +CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/sglang_sm120_cache}" + +# Matrix contains ISL=1K..128K with fixed OSL=1K. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" + +# PyTorch CUDA allocator setting for the SGLang server. expandable_segments +# reduces fragmentation from the GiB-scale indexer temporaries that OOM the +# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096. +PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/matrix.json b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/matrix.json new file mode 100644 index 0000000..ea468ea --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/matrix.json @@ -0,0 +1,14 @@ +{ + "comment": "Tiny adaptive test: fixed OSL=1024, ISL=1K..128K. Only Y entries are run.", + "mode": "Y", + "matrix": { + "1024": {"1024": "Y"}, + "2048": {"1024": "Y"}, + "4096": {"1024": "Y"}, + "8192": {"1024": "Y"}, + "16384": {"1024": "Y"}, + "32768": {"1024": "Y"}, + "65536": {"1024": "Y"}, + "131072": {"1024": "Y"} + } +} diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..2d993e1 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/run_adaptive_concurrency_add16.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# Tiny test: find Total-TPS saturation for fixed OSL=1K and ISL=1K..128K. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + python3 -m sglang.launch_server --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then + : # random-ids does not need --tokenize-prompt + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + --entrypoint python3 \ + "$DOCKER_IMAGE" \ + -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" + else + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 +# If the initial concurrency violates the TTFT SLO, search downward. Stop at +# the first acceptable value (16 -> 8; only try 1 when 8 still violates it). +export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1" +# When concurrency 1 still has a severely excessive TTFT, stop the remaining +# shapes in this TP/DP group. Zero disables this rule. +export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}" + +adaptive_main "$@" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh new file mode 100755 index 0000000..e251dd5 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Start SGLang server in Docker for a given TPxDP configuration. +# Usage: start_sglang_docker.sh +# +# Uses the verified SGLang SM120 image and keeps its JIT cache on +# persistent storage. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR" + +IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + -m sglang.launch_server + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: python3 ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --privileged \ + --ipc=host \ + --network host \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --entrypoint python3 \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}:/root/.cache" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh new file mode 100755 index 0000000..b7e69e3 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the +# local VENV_CLIENT environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_CLIENT}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --host 0.0.0.0 + --port "$SGLANG_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${SGLANG_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/adaptive_config.env new file mode 100644 index 0000000..cdf35c2 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/adaptive_config.env @@ -0,0 +1,56 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/config.env b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/config.env new file mode 100644 index 0000000..0860a6b --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/config.env @@ -0,0 +1,78 @@ +# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D +# (8 GPUs) using vLLM. It fixes OSL=1024 and sweeps ISL=1K..128K. +# Tests vLLM with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_pro6000_vllm_tiny_1k_output" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/6000D/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +VLLM_PORT="${VLLM_PORT:-30030}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# vLLM server settings, verified for this model and machine. +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +BLOCK_SIZE="${BLOCK_SIZE:-256}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" + +# Deployment switch. 0 = native vllm venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" + +# Benchmark client Docker image. vLLM's image does not include the SGLang benchmark client, +# so the client runs inside the SGLang image and targets the vLLM backend via the +# OpenAI-compatible API. +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" + +# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}" +SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}" +CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/vllm_sm120_cache}" + +# Matrix contains ISL=1K..128K with fixed OSL=1K. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/matrix.json b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/matrix.json new file mode 100644 index 0000000..ea468ea --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/matrix.json @@ -0,0 +1,14 @@ +{ + "comment": "Tiny adaptive test: fixed OSL=1024, ISL=1K..128K. Only Y entries are run.", + "mode": "Y", + "matrix": { + "1024": {"1024": "Y"}, + "2048": {"1024": "Y"}, + "4096": {"1024": "Y"}, + "8192": {"1024": "Y"}, + "16384": {"1024": "Y"}, + "32768": {"1024": "Y"}, + "65536": {"1024": "Y"}, + "131072": {"1024": "Y"} + } +} diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..23589de --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/run_adaptive_concurrency_add16.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Tiny test: find Total-TPS saturation for fixed OSL=1K and ISL=1K..128K. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then + : # random-ids does not need --tokenize-prompt + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + --entrypoint python3 \ + "$DOCKER_CLIENT_IMAGE" \ + -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" + else + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 +# If the initial concurrency violates the TTFT SLO, search downward. Stop at +# the first acceptable value (16 -> 8; only try 1 when 8 still violates it). +export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1" +# When concurrency 1 still has a severely excessive TTFT, stop the remaining +# shapes in this TP/DP group. Zero disables this rule. +export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}" + +adaptive_main "$@" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh new file mode 100755 index 0000000..cd07631 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Start vLLM server in Docker for a given TPxDP configuration. +# Usage: start_vllm_docker.sh +# +# Uses the verified vLLM SM120 image and keeps its cache on persistent +# storage. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR" + +IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" +PORT="${VLLM_PORT:-30030}" +NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: vllm ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --privileged \ + --ipc=host \ + --network host \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --entrypoint vllm \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}:/root/.cache" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 600); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + # Check if the Docker container is still running (not the nohup PID). + # Allow a brief grace period for the container to appear in docker ps. + container_running=0 + for _ in $(seq 1 3); do + if docker ps --filter "name=${NAME}" --format '{{.Names}}' | grep -q "^${NAME}$"; then + container_running=1 + break + fi + sleep 1 + done + if [[ "$container_running" -eq 0 ]]; then + echo "ERROR: Docker vLLM container exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/600)" + sleep 5 +done + +echo "ERROR: Docker vLLM server not healthy after 600 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh new file mode 100755 index 0000000..d652590 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Start vLLM server for a given TP×DP configuration. +# Usage: start_vllm_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the +# local VENV_VLLM environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --host 0.0.0.0 + --port "$VLLM_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${VLLM_PORT}..." + +for i in $(seq 1 600); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/600)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 600 retries" +tail -200 "$LOG" +exit 1