diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py index bdd3224..b8d259b 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py @@ -22,10 +22,10 @@ def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, ttft_ok = ttft_p95_ms < ttft_limit_ms tpot_ok = tpot_mean_ms < tpot_limit_ms if ttft_ok and tpot_ok: - return "✅" + return "PASS" if ttft_ok or tpot_ok: - return "⚠️" - return "❌" + return "PARTIAL" + return "FAIL" def gpu_memory_str(gpu: dict | None) -> str: @@ -82,7 +82,7 @@ def main(): f.write("## Summary\n\n") f.write(f"- Model: `{model}`\n") f.write(f"- Hardware: {hardware}\n") - f.write("- Backend: vLLM\n") + f.write("- Backend: vLLM (Docker)\n") f.write("- Benchmark client: `sglang.bench_serving`\n") f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") @@ -152,7 +152,7 @@ def main(): f.write("\n## Notes\n\n") f.write("- SLO check uses TTFT P95 and TPOT mean.\n") - f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 61b8be2..246c1e1 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -1,4 +1,4 @@ -# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs). +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using vLLM. # Tests vLLM with three parallel configurations: # TP=2, DP=4 -> 2 GPUs per replica, 4 replicas # TP=4, DP=2 -> 4 GPUs per replica, 2 replicas @@ -6,18 +6,25 @@ EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" +MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" VLLM_PORT="${VLLM_PORT:-30030}" -VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" -VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" -# The benchmark client is sglang.bench_serving, even when the backend is vLLM. -VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + # Parallel configurations to test. Format: "TP DP" declare -a PARALLEL_CONFIGS=( "2 4" @@ -25,14 +32,34 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) +# vLLM server settings +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +BLOCK_SIZE="${BLOCK_SIZE:-256}" + +# Deployment switch. 0 = native vllm venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" + +# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving, +# so the client runs inside the SGLang image and targets the vLLM backend via the +# OpenAI-compatible API. +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H200 nodes. +DATASET_PATH="${DATASET_PATH:-${SCRIPT_DIR}/runtime/dummy_sharegpt.json}" + # Matrix and concurrency rules are defined in matrix.json by default. -# This file is consumed by generate_scenarios.py. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" # Sampling density for concurrency. -# 0 = use the default heuristic in generate_scenarios.py (6-8 points). -# 2 = only test the two endpoints (low and high). +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" # Per-scenario timeout to avoid hangs (seconds). @@ -47,8 +74,3 @@ DRY_RUN="${DRY_RUN:-0}" # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. GRID_LIMIT="${GRID_LIMIT:-0}" - -# Docker deployment switch. If 1, vLLM is launched via Docker using the image -# below instead of the local VENV_VLLM environment. -USE_DOCKER="${USE_DOCKER:-0}" -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py index ef46e01..f722479 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py @@ -15,23 +15,16 @@ from pathlib import Path def sample_concurrency(low: int, high: int, target: int) -> list[int]: - """Return a sorted list of concurrency values in [low, high]. + """Return only the low and high concurrency values in [low, high]. - If the range is small, return every integer. Otherwise sample roughly - `target` points linearly between low and high (inclusive). + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. """ assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" - if high - low + 1 <= target: - return list(range(low, high + 1)) - - points = set() - points.add(low) - points.add(high) - step = (high - low) / (target - 1) - for i in range(1, target - 1): - v = low + round(step * i) - points.add(max(low, min(high, v))) - return sorted(points) + if low == high: + return [low] + return [low, high] def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index 99e8de3..f004020 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM. +# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM (Docker). set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -12,6 +12,10 @@ source "${SCRIPT_DIR}/../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE \ + DOCKER_IMAGE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT + RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" @@ -21,7 +25,14 @@ GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" DRY_RUN="${DRY_RUN:-0}" GRID_LIMIT="${GRID_LIMIT:-0}" -PYTHON="${VENV_CLIENT}/bin/python" +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" mkdir -p "$log_dir_global" @@ -42,7 +53,7 @@ stop_server() { local tp="$1" local dp="$2" - local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" if [[ -f "$pid_file" ]]; then local pid pid="$(cat "$pid_file")" @@ -55,6 +66,9 @@ stop_server() { rm -f "$pid_file" fi + # Fallback: remove any Docker container started by this experiment. + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + # Fallback: kill any vllm serve processes for this model. pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true @@ -68,10 +82,14 @@ build_server_args() { local args=( "vllm serve" "$MODEL_PATH" --trust-remote-code - --kv-cache-dtype fp8 - --block-size 256 + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune + --host 0.0.0.0 --port "$VLLM_PORT" ) if [[ "$dp" -gt 1 ]]; then @@ -106,20 +124,51 @@ restart_server() { start_server "$tp" "$dp" } +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" && -f "${DATASET_PATH}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") + docker run --rm \ + --network host \ + "${vol_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e HF_DATASETS_OFFLINE=1 \ + "${DOCKER_CLIENT_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} +export -f run_bench_serving + run_warmup() { local input_len="$1" local output_len="$2" log "warming up (input=${input_len}, output=${output_len}, num=1)" - "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ - --backend vllm \ - --host 127.0.0.1 \ - --port "$VLLM_PORT" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num 1 \ - --env-python "$PYTHON" \ - >> "${log_dir_global}/warmup.log" 2>&1 + bash -c ' + run_bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "'"$VLLM_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' log "warmup completed" } @@ -307,7 +356,7 @@ run_parallel_config() { "$ACCELERATOR" \ "$CHIP" \ "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_VLLM" \ + "$DOCKER_IMAGE" \ "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" local server_args_str @@ -383,20 +432,22 @@ run_parallel_config() { gpu_pid="$(start_gpu_monitor "$gpu_csv")" bench_rc=0 - timeout "$SCENARIO_TIMEOUT_S" \ - "$PYTHON" -m sglang.bench_serving \ + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ --backend vllm \ --host 127.0.0.1 \ - --port "$VLLM_PORT" \ + --port "'"$VLLM_PORT"'" \ --dataset-name random \ - --random-input-len "$isl" \ - --random-output-len "$dsl" \ - --num-prompts "$num" \ - --max-concurrency "$conc" \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$dsl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ --request-rate 10000 \ - --output-file "$output_file" \ + --output-file "'"$output_file"'" \ --output-details \ - > "$detail_log" 2>&1 || bench_rc=$? + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? stop_gpu_monitor "$gpu_pid" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh index ae7f728..e133fd7 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -2,7 +2,7 @@ # Start vLLM server in Docker for a given TPxDP configuration. # Usage: start_vllm_docker.sh # -# Uses the vllm/vllm-openai image and the same minimal argument set as the +# Uses the vllm/vllm-openai image and the same argument set as the # bare-metal start script. The container is removed automatically on stop. set -e @@ -13,15 +13,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -cd /data/user1/yy -mkdir -p logs +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" PORT="${VLLM_PORT:-30030}" NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" -PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" -LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" rm -f "$PID_FILE" # Clean up any stale container with the same name. @@ -30,10 +30,14 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( serve "$MODEL_PATH" --trust-remote-code - --kv-cache-dtype bfloat16 - --block-size 256 + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune + --host 0.0.0.0 --port "$PORT" ) @@ -50,7 +54,7 @@ echo "Image: $IMAGE" echo "Model: $MODEL_PATH" echo "Container name: $NAME" echo "Host port: $PORT" -echo "Command: vllm $SERVER_ARGS_STR" +echo "Command: vllm ${SERVER_ARGS_STR}" echo "Log: $LOG" # Run docker in the foreground so that killing the host process stops the @@ -59,12 +63,17 @@ echo "Log: $LOG" nohup docker run --rm \ --name "$NAME" \ --gpus all \ + --ipc host \ + --shm-size 16g \ + --ulimit memlock=-1 \ --entrypoint vllm \ -p "${PORT}:${PORT}" \ - -v /data/models:/data/models:ro \ - -v /data/user1/yy/tmp:/tmp \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ "$IMAGE" \ "${SERVER_ARGS[@]}" \ > "$LOG" 2>&1 & diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh index 77b662a..741ca9c 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -2,14 +2,9 @@ # Start vLLM server for a given TP×DP configuration. # Usage: start_vllm_dp.sh # -# Uses the minimal argument set from the official DeepSeek-V4-Flash recipe: -# --trust-remote-code -# --kv-cache-dtype fp8 -# --block-size 256 -# --tensor-parallel-size -# --no-enable-flashinfer-autotune -# When DP > 1, only --data-parallel-size is added so vLLM uses its internal -# DP load balancer (single API server, single port). +# By default this delegates to the Docker start script because the experiment +# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the +# local VENV_VLLM environment instead. set -e TP="${1}" @@ -19,38 +14,42 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" -if [[ "${USE_DOCKER:-0}" == "1" ]]; then +if [[ "${USE_DOCKER:-1}" == "1" ]]; then exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" fi -cd /data/user1/yy -mkdir -p logs +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" VENV="${VENV_VLLM}" export PATH="$VENV/bin:$PATH" export PYTHONUNBUFFERED=1 -export TMPDIR=/data/user1/yy/tmp +export TMPDIR="${RUNTIME_BASE}/tmp" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" -LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" SERVER_ARGS=( - vllm serve "$MODEL_PATH" - --trust-remote-code - --kv-cache-dtype fp8 - --block-size 256 - --tensor-parallel-size "$TP" - --no-enable-flashinfer-autotune - --port "$VLLM_PORT" + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" ) if [[ "$DP" -gt 1 ]]; then - SERVER_ARGS+=( - --data-parallel-size "$DP" - ) + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) fi SERVER_ARGS_STR="${SERVER_ARGS[*]}" @@ -69,18 +68,18 @@ echo "PID: $PID" echo "Waiting for health on port ${VLLM_PORT}..." for i in $(seq 1 240); do - if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then - echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: vLLM server exited early" - tail -200 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 done echo "ERROR: vLLM server not healthy after 240 retries"