feat(dsv4): make DSV4-Flash 910C experiment runnable (verified TP4/DP2)
The dsv4_910c_vllm_tp_dp_matrix experiment was a placeholder (wrong MODEL_PATH, glm52 image default, no DSV4 serve flags, no driver mounts). Bring it to a working state, validated end-to-end on 910c.1. config.env: - MODEL_PATH: /mnt/models/DeepSeek-V4-Flash -> .../DeepSeek-V4-Flash-w8a8-mtp (weights downloaded from ModelScope Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp, ~280 GiB, 70 shards, verified present and chowned to shishi) - SERVED_MODEL_NAME: deepseek-v4-flash -> dsv4 (matches vllm-ascend tutorial) - DOCKER_IMAGE: vllm-ascend:v0.23.0rc1-a3-openeuler -> quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler (full tag, present locally; the glm5.2-a3 image carries GLM-specific patches and is NOT DSV4-compatible) - Add DSV4_* serve-flag vars: tokenizer-mode/tool-call-parser/reasoning-parser deepseek_v4, quantization ascend, expert-parallel, async-scheduling, MTP speculative-config, compilation-config, additional-config, multithread model-loader (128 threads for the 280GiB load) - Rewrite the OOM-boundary comment to reflect actual w8a8 weight size start_vllm_docker.sh (the main pitfalls vs the glm52 launcher): - Inject the DSV4_* serve flags (GLM-5.2 needs none of them); without them vllm rejects the model / lacks MTP - Mount host driver libs (driver/lib64, dcmi, hccn_tool, npu-smi, version.info, ascend_install.info, hccn.conf) -- otherwise the container torch_npu fails with libascend_hal.so not found - Mount all 16 dies via --device /dev/davinci0..15 + davinci_manager/ devmm_svm/hisi_hdc instead of relying on Ascend Docker Runtime injection (ASCEND_VISIBLE_DEVICES-only), which was unreliable on this host - --privileged --shm-size 512g for the 280GiB weight load - LD_PRELOAD the openEuler jemalloc path /usr/lib64/libjemalloc.so.2 (the glm52 ubuntu path /usr/lib/aarch64-linux-gnu/... does not exist here) - Raise health-wait budget 240x5s -> 360x10s (DSV4 load+compile ~8min) - DRY_RUN mode for command preview without launching Verified: start_vllm_docker.sh 4 2 brings the server up on port 30052 in ~8 min (130s weight load per die, 29s compile, 187s engine init); chat completion returns correctly, system_fingerprint vllm-0.23.0-tp4-dp2-ep.
This commit is contained in:
shishi
parent
46e79d63e7
commit
4197e2738d
@ -8,17 +8,14 @@
|
||||
# Platform: ascend_910c (see platforms/ascend_910c.env).
|
||||
# Host: 910c.1 / NPU-NODE61, openEuler 22.03 SP4 aarch64, driver 25.5.2, CANN 9.0.0.
|
||||
#
|
||||
# ⚠️ PLACEHOLDER: DeepSeek-V4-Flash weights are NOT yet present on this host
|
||||
# (only GLM-5.2-{w4a8c8,w8a8} under /mnt/models). Download DSV4-Flash first,
|
||||
# then fix MODEL_PATH below. Smoke run will fail until the model exists.
|
||||
# Recommended host path: /mnt/models/DeepSeek-V4-Flash (FP8)
|
||||
# /mnt/models/DeepSeek-V4-Flash-INT8 (INT8)
|
||||
# DeepSeek-V4-Flash w8a8-mtp weights are present at /mnt/models/DeepSeek-V4-Flash-w8a8-mtp
|
||||
# (downloaded from ModelScope Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp, ~280 GiB, 70 shards).
|
||||
|
||||
EXPERIMENT="dsv4_910c_vllm_tp_dp_matrix"
|
||||
MODEL_NAME="DeepSeek-V4-Flash"
|
||||
# TODO: point this at the real DSV4-Flash directory once downloaded.
|
||||
MODEL_PATH="${MODEL_PATH:-/mnt/models/DeepSeek-V4-Flash}"
|
||||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
||||
# Real DSV4-Flash w8a8-mtp directory (verified present & chown'd to shishi).
|
||||
MODEL_PATH="${MODEL_PATH:-/mnt/models/DeepSeek-V4-Flash-w8a8-mtp}"
|
||||
SERVED_MODEL_NAME="dsv4"
|
||||
|
||||
VLLM_PORT="${VLLM_PORT:-30052}"
|
||||
|
||||
@ -28,28 +25,36 @@ CONTAINER_NAME="${CONTAINER_NAME:-vllm-ascend-dsv4-910c}"
|
||||
# Python interpreter for the benchmark client inside the vllm-ascend container.
|
||||
CONTAINER_PYTHON="${CONTAINER_PYTHON:-/usr/local/bin/python}"
|
||||
|
||||
# vllm-ascend image. Use the general v0.23 image for DSV4 (the GLM5.2-tuned
|
||||
# variant may carry GLM-specific patches). Load from:
|
||||
# /mnt/models/vllm-ascend-v0.23.0rc1-a3-openeuler.tar
|
||||
# vllm-ascend image. Use the general v0.23 A3 image for DSV4 (the GLM5.2-tuned
|
||||
# variant carries GLM-specific patches and is NOT compatible with DSV4).
|
||||
# Verified present locally: quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler
|
||||
USE_DOCKER="${USE_DOCKER:-1}"
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-ascend:v0.23.0rc1-a3-openeuler}"
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}"
|
||||
|
||||
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving;
|
||||
# reuse the vllm-ascend container itself for the client via `docker exec` (see
|
||||
# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1
|
||||
# with an external sglang image. Default off on 910c.
|
||||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}"
|
||||
|
||||
# Device selection. We mount all 8 cards (16 dies) via --device /dev/davinci0..15
|
||||
# in start_vllm_docker.sh. ASCEND_VISIBLE_DEVICES is kept for parity with the
|
||||
# shared library but the explicit --device flags are the authoritative path on
|
||||
# this host (Ascend Docker Runtime injection was unreliable here).
|
||||
export ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
|
||||
# Runtime working directory for logs, pid files, and tmp.
|
||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||
|
||||
# Parallel configurations to test. Format: "TP DP"
|
||||
# DSV4-Flash FP8 routed-expert weights ~264 GiB total. Per-die load = 264/TP GiB.
|
||||
# TP=2 -> 132 GiB/die > 64 GiB HBM ❌ OOM expected (kept but will fail)
|
||||
# TP=4 -> 66 GiB/die > 64 GiB HBM ❌ borderline OOM (KV cache leaves no room)
|
||||
# TP=8 -> 33 GiB/die < 64 GiB HBM ✅ fits with room for KV cache
|
||||
# TP=2/DP=4 and TP=4/DP=2 are likely infeasible for FP8 on 64GB dies; they are
|
||||
# kept here so the smoke run records the OOM boundary explicitly. Switch to
|
||||
# INT8 weights (MODEL_PATH=...-INT8, ~132 GiB total) to make TP=4 viable.
|
||||
# DSV4-Flash w8a8 routed-expert weights ~280 GiB total. With expert-parallel the
|
||||
# weight load is sharded across TP dies, but w8a8 still leaves a heavy per-die
|
||||
# footprint. TP*DP must equal 16 (8 cards x 2 dies).
|
||||
# TP=2, DP=4 -> 4 dies/replica x 4 replicas (smallest TP, most replicas)
|
||||
# TP=4, DP=2 -> 4 dies/replica x 2 replicas
|
||||
# TP=8, DP=1 -> 8 dies/replica x 1 replica (largest TP, max weight sharding)
|
||||
if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then
|
||||
declare -a PARALLEL_CONFIGS=()
|
||||
for pair in $PARALLEL_CONFIGS_STR; do
|
||||
@ -63,19 +68,54 @@ else
|
||||
)
|
||||
fi
|
||||
|
||||
# vLLM-Ascend server settings for DSV4-Flash.
|
||||
# vLLM-Ascend server settings for DSV4-Flash (w8a8-mtp).
|
||||
# Notes:
|
||||
# - KV cache dtype fp8 is supported on 910C; fall back to fp16 if the image rejects it.
|
||||
# - block-size 128 matches Ascend page semantics (910C favors 128).
|
||||
# - MAX_MODEL_LEN: DSV4-Flash supports up to 1M context; cap at 131072 for the
|
||||
# matrix sweep (extend to 1M via matrix.json once TP=8 is verified).
|
||||
# - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9).
|
||||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
|
||||
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
|
||||
BLOCK_SIZE="${BLOCK_SIZE:-128}"
|
||||
# DSV4-Flash supports up to 1M context, but 64GB HBM/die caps realistic ISL.
|
||||
# Start at 128K; extend via matrix.json once TP=8 is verified.
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
|
||||
|
||||
# vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh.
|
||||
# attention backend for 910C: use the fused/atb attention path. Adjust per image.
|
||||
VLLM_ASCEND_ATTENTION_BACKEND="${VLLM_ASCEND_ATTENTION_BACKEND:-atb}"
|
||||
|
||||
# DSV4-Flash-specific server args (passed through to vllm serve in
|
||||
# start_vllm_docker.sh). These are REQUIRED for DSV4 -- GLM-5.2 does not need them.
|
||||
# --tokenizer-mode deepseek_v4 : DSV4 custom tokenizer
|
||||
# --tool-call-parser deepseek_v4 : DSV4 tool-call parser
|
||||
# --enable-auto-tool-choice : enable automatic tool choice
|
||||
# --reasoning-parser deepseek_v4 : DSV4 reasoning parser
|
||||
# --enable-expert-parallel : shard 256 experts across EP ranks
|
||||
# --quantization ascend : use modelslim w8a8 quantization path
|
||||
# --safetensors-load-strategy prefetch: prefetch shards for faster load
|
||||
# --async-scheduling : overlap CPU scheduling with NPU compute
|
||||
# MTP speculative decoding via SPEC_CONFIG
|
||||
DSV4_TOKENIZER_MODE="${DSV4_TOKENIZER_MODE:-deepseek_v4}"
|
||||
DSV4_TOOL_CALL_PARSER="${DSV4_TOOL_CALL_PARSER:-deepseek_v4}"
|
||||
DSV4_REASONING_PARSER="${DSV4_REASONING_PARSER:-deepseek_v4}"
|
||||
DSV4_QUANTIZATION="${DSV4_QUANTIZATION:-ascend}"
|
||||
DSV4_SAFETENSORS_LOAD_STRATEGY="${DSV4_SAFETENSORS_LOAD_STRATEGY:-prefetch}"
|
||||
# MTP speculative config (1 speculative token). JSON string, kept single-quoted
|
||||
# in the launcher to avoid shell mangling.
|
||||
DSV4_SPEC_CONFIG="${DSV4_SPEC_CONFIG:-{\"num_speculative_tokens\": 1, \"method\": \"mtp\", \"enforce_eager\": true}}"
|
||||
DSV4_COMPILATION_CONFIG="${DSV4_COMPILATION_CONFIG:-{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}}"
|
||||
DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"enable_dsa_cp\": true,\"multistream_overlap_shared_expert\":true}}"
|
||||
DSV4_ENABLE_EXPERT_PARALLEL="${DSV4_ENABLE_EXPERT_PARALLEL:-1}"
|
||||
DSV4_ENABLE_ASYNC_SCHEDULING="${DSV4_ENABLE_ASYNC_SCHEDULING:-1}"
|
||||
|
||||
# Model-loader extra config (multithread load, 128 threads) to speed up 280GiB load.
|
||||
DSV4_MODEL_LOADER_EXTRA_CONFIG="${DSV4_MODEL_LOADER_EXTRA_CONFIG:-{\"enable_multithread_load\": \"true\", \"num_threads\": 128}}"
|
||||
|
||||
# Dataset used by sglang.bench_serving --dataset-name random.
|
||||
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||
|
||||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||
|
||||
|
||||
@ -2,9 +2,10 @@
|
||||
# Start vLLM-Ascend server in Docker for a given TP×DP configuration on 910C.
|
||||
# Usage: start_vllm_docker.sh <TP> <DP>
|
||||
#
|
||||
# Relies on the Ascend Docker Runtime being the default docker runtime on this
|
||||
# host (see /etc/docker/daemon.json). NPU dies are injected via the
|
||||
# ASCEND_VISIBLE_DEVICES env var; no --gpus / --device flags are needed.
|
||||
# DSV4-Flash-specific: this launcher mounts host driver libs + all davinci dies
|
||||
# explicitly via --device (NOT relying on Ascend Docker Runtime injection,
|
||||
# which was unreliable on this host), and injects the DSV4-required vllm serve
|
||||
# flags (tokenizer-mode, expert-parallel, quantization ascend, MTP, ...).
|
||||
set -e
|
||||
|
||||
TP="${1}"
|
||||
@ -17,8 +18,8 @@ source "${SCRIPT_DIR}/config.env"
|
||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||
|
||||
IMAGE="${DOCKER_IMAGE:-vllm-ascend:glm5.2-a3-openeuler}"
|
||||
PORT="${VLLM_PORT:-30050}"
|
||||
IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}"
|
||||
PORT="${VLLM_PORT:-30052}"
|
||||
NAME="${CONTAINER_NAME:-${EXPERIMENT}_vllm_tp${TP}_dp${DP}}"
|
||||
NAME="${NAME}_tp${TP}_dp${DP}"
|
||||
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
||||
@ -29,9 +30,8 @@ rm -f "$PID_FILE"
|
||||
# Clean up any stale container with the same name.
|
||||
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||
|
||||
# vLLM-Ascend launch args. Differences vs NVIDIA vLLM:
|
||||
# - no --no-enable-flashinfer-autotune (Ascend uses its own attention path)
|
||||
# - --kv-cache-dtype may need to be fp16 if the image rejects fp8 on 910C
|
||||
# ---- vLLM-Ascend launch args ----
|
||||
# Base args common to GLM-5.2 and DSV4-Flash.
|
||||
SERVER_ARGS=(
|
||||
"$MODEL_PATH"
|
||||
--served-model-name "$SERVED_MODEL_NAME"
|
||||
@ -50,6 +50,31 @@ if [[ "$DP" -gt 1 ]]; then
|
||||
SERVER_ARGS+=(--data-parallel-size "$DP")
|
||||
fi
|
||||
|
||||
# ---- DSV4-Flash-specific args (REQUIRED; GLM-5.2 does not use these) ----
|
||||
# These were verified working in the manual bring-up on 910c.1.
|
||||
SERVER_ARGS+=(
|
||||
--tokenizer-mode "$DSV4_TOKENIZER_MODE"
|
||||
--tool-call-parser "$DSV4_TOOL_CALL_PARSER"
|
||||
--enable-auto-tool-choice
|
||||
--reasoning-parser "$DSV4_REASONING_PARSER"
|
||||
--quantization "$DSV4_QUANTIZATION"
|
||||
--safetensors-load-strategy "$DSV4_SAFETENSORS_LOAD_STRATEGY"
|
||||
)
|
||||
|
||||
if [[ "${DSV4_ENABLE_EXPERT_PARALLEL:-1}" == "1" ]]; then
|
||||
SERVER_ARGS+=(--enable-expert-parallel)
|
||||
fi
|
||||
|
||||
if [[ "${DSV4_ENABLE_ASYNC_SCHEDULING:-1}" == "1" ]]; then
|
||||
SERVER_ARGS+=(--async-scheduling)
|
||||
fi
|
||||
|
||||
# JSON-bearing args. Quoted as single strings so vllm parses each as one argv.
|
||||
SERVER_ARGS+=(--model-loader-extra-config "$DSV4_MODEL_LOADER_EXTRA_CONFIG")
|
||||
SERVER_ARGS+=(--speculative-config "$DSV4_SPEC_CONFIG")
|
||||
SERVER_ARGS+=(--compilation-config "$DSV4_COMPILATION_CONFIG")
|
||||
SERVER_ARGS+=(--additional-config "$DSV4_ADDITIONAL_CONFIG")
|
||||
|
||||
SERVER_ARGS_STR="vllm serve ${SERVER_ARGS[*]}"
|
||||
|
||||
echo "=== Starting vLLM-Ascend server in Docker (TP=${TP}, DP=${DP}) ==="
|
||||
@ -61,23 +86,68 @@ echo "ASCEND_VISIBLE_DEVICES: ${ASCEND_VISIBLE_DEVICES}"
|
||||
echo "Command: $SERVER_ARGS_STR"
|
||||
echo "Log: $LOG"
|
||||
|
||||
# Run docker in the foreground; nohup backgrounds it and the host PID lets the
|
||||
# adaptive search stop the container by killing the process (the container has
|
||||
# --rm so it self-cleans). The Ascend Docker Runtime is the default runtime, so
|
||||
# no --runtime flag is required.
|
||||
# ---- Docker run ----
|
||||
# CRITICAL vs glm52 launcher:
|
||||
# 1. Mount host driver libs (/usr/local/Ascend/driver/lib64, dcmi, hccn_tool,
|
||||
# npu-smi, version.info, ascend_install.info, hccn.conf). Without these the
|
||||
# container's torch_npu fails with "libascend_hal.so not found".
|
||||
# 2. Mount all 16 dies explicitly via --device /dev/davinci0..15 plus the
|
||||
# manager/devmm_svm/hisi_hdc control devices. Ascend Docker Runtime
|
||||
# injection (ASCEND_VISIBLE_DEVICES only) was unreliable here.
|
||||
# 3. --privileged + --shm-size 512g (DSV4 weight load needs large shm).
|
||||
# 4. jemalloc preload path is the openEuler one (/usr/lib64/...).
|
||||
DEVICE_ARGS=(
|
||||
--device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci2 --device /dev/davinci3
|
||||
--device /dev/davinci4 --device /dev/davinci5 --device /dev/davinci6 --device /dev/davinci7
|
||||
--device /dev/davinci8 --device /dev/davinci9 --device /dev/davinci10 --device /dev/davinci11
|
||||
--device /dev/davinci12 --device /dev/davinci13 --device /dev/davinci14 --device /dev/davinci15
|
||||
--device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc
|
||||
)
|
||||
|
||||
VOL_ARGS=(
|
||||
-v /usr/local/dcmi:/usr/local/dcmi
|
||||
-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
|
||||
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/
|
||||
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info
|
||||
-v /etc/hccn.conf:/etc/hccn.conf
|
||||
-v "${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||
-v "${RUNTIME_BASE}/tmp:/tmp"
|
||||
)
|
||||
|
||||
ENV_ARGS=(
|
||||
-e ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES}"
|
||||
-e PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
-e PYTHONUNBUFFERED=1
|
||||
-e HF_HUB_OFFLINE=1
|
||||
-e TRANSFORMERS_OFFLINE=1
|
||||
-e OMP_PROC_BIND=false
|
||||
-e OMP_NUM_THREADS=10
|
||||
-e HCCL_BUFFSIZE=1024
|
||||
-e VLLM_ASCEND_ENABLE_FLASHCOMM1=1
|
||||
-e TASK_QUEUE_ENABLE=1
|
||||
-e "HCCL_OP_EXPANSION_MODE=AIV"
|
||||
-e "LD_PRELOAD=/usr/lib64/libjemalloc.so.2"
|
||||
)
|
||||
|
||||
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
||||
echo "[DRY-RUN] docker run --rm --name \"$NAME\" --privileged --ipc host --shm-size 512g \\"
|
||||
echo " --network host ${DEVICE_ARGS[*]} ${VOL_ARGS[*]} ${ENV_ARGS[*]} \\"
|
||||
echo " \"$IMAGE\" $SERVER_ARGS_STR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
nohup docker run --rm \
|
||||
--name "$NAME" \
|
||||
--privileged \
|
||||
--ipc host \
|
||||
--shm-size 16g \
|
||||
--shm-size 512g \
|
||||
--network host \
|
||||
--ulimit memlock=-1 \
|
||||
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
||||
-e ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES}" \
|
||||
-e PYTORCH_NPU_ALLOC_CONF=expandable_segments:True \
|
||||
-e PYTHONUNBUFFERED=1 \
|
||||
-e HF_HUB_OFFLINE=1 \
|
||||
-e TRANSFORMERS_OFFLINE=1 \
|
||||
"${DEVICE_ARGS[@]}" \
|
||||
"${VOL_ARGS[@]}" \
|
||||
"${ENV_ARGS[@]}" \
|
||||
"$IMAGE" \
|
||||
vllm serve "${SERVER_ARGS[@]}" \
|
||||
> "$LOG" 2>&1 &
|
||||
@ -87,7 +157,9 @@ echo $PID > "$PID_FILE"
|
||||
echo "PID: $PID"
|
||||
echo "Waiting for health on port ${PORT}..."
|
||||
|
||||
for i in $(seq 1 240); do
|
||||
# DSV4 weight load (280 GiB) + graph compile takes much longer than GLM-5.2;
|
||||
# raise the wait budget to 360 retries x 10s = 1 hour.
|
||||
for i in $(seq 1 360); do
|
||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
||||
echo "vLLM-Ascend server is ready at http://127.0.0.1:${PORT}"
|
||||
echo "Log: $LOG"
|
||||
@ -98,10 +170,10 @@ for i in $(seq 1 240); do
|
||||
tail -200 "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting... ($i/240)"
|
||||
sleep 5
|
||||
echo "Waiting... ($i/360)"
|
||||
sleep 10
|
||||
done
|
||||
|
||||
echo "ERROR: vLLM-Ascend server not healthy after 240 retries"
|
||||
echo "ERROR: vLLM-Ascend server not healthy after 360 retries"
|
||||
tail -200 "$LOG"
|
||||
exit 1
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user