diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env index 52ac44c..9c92a74 100644 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env @@ -20,16 +20,16 @@ SERVED_MODEL_NAME="dsv4" VLLM_PORT="${VLLM_PORT:-30052}" # Dedicated container name so this experiment never touches other 910c runs. -CONTAINER_NAME="${CONTAINER_NAME:-vllm-ascend-dsv4-910c}" +CONTAINER_NAME="vllm-ascend-dsv4-910c" # Python interpreter for the benchmark client inside the vllm-ascend container. -CONTAINER_PYTHON="${CONTAINER_PYTHON:-/usr/local/bin/python}" +CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3" # vllm-ascend image. Use the general v0.23 A3 image for DSV4 (the GLM5.2-tuned # variant carries GLM-specific patches and is NOT compatible with DSV4). # Verified present locally: quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler USE_DOCKER="${USE_DOCKER:-1}" -DOCKER_IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}" +DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}" # Benchmark client Docker image. vLLM's image does not include sglang.bench_serving; # reuse the vllm-ascend container itself for the client via `docker exec` (see diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index e5d87eb..3242291 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -97,6 +97,8 @@ engine_start_server() { local tp="$1" local dp="$2" local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + export ENGINE_TP="$tp" + export ENGINE_DP="$dp" log "starting vllm-ascend server tp=${tp} dp=${dp}" bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 if ! engine_is_healthy; then @@ -117,6 +119,8 @@ engine_detect_oom() { local dp="$3" local pattern='out of memory|OutOfMemory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory|NPU out of memory|acl.*memory|HBM' local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + export ENGINE_TP="$tp" + export ENGINE_DP="$dp" local -a logs=("$detail_log" "$outer_log") if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then logs+=("$ACTIVE_ENGINE_SERVER_LOG") @@ -130,7 +134,7 @@ engine_run_bench() { local concurrency="$3" local num_prompts="$4" local output_file="$5" - local container_name="${CONTAINER_NAME}_tp${tp}_dp${dp}" + local container_name="${CONTAINER_NAME}_tp${ENGINE_TP:-${tp}}_dp${ENGINE_DP:-${dp}}" local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")" if [[ "$output_file" == "/dev/null" ]]; then container_output="/dev/null" @@ -143,7 +147,7 @@ engine_run_bench() { --backend vllm --host 127.0.0.1 --port "$ENGINE_PORT" - --model "$SERVED_MODEL_NAME" + --tokenizer "$MODEL_PATH" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" --random-output-len "$osl" @@ -170,7 +174,7 @@ engine_run_bench() { # host port. See envs/ASCEND_910C_ENV_SETUP.md. docker exec "$container_name" mkdir -p /tmp/bench_outputs 2>/dev/null || true docker exec "$container_name" \ - env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 TORCH_DEVICE_BACKEND_AUTOLOAD=0 \ "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $? if [[ "$output_file" != "/dev/null" ]]; then @@ -179,7 +183,7 @@ engine_run_bench() { } export -f engine_run_bench -export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE SERVED_MODEL_NAME +export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE SERVED_MODEL_NAME ENGINE_TP ENGINE_DP export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON export SEARCH_START_CONCURRENCY=16 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index c63d3a2..f17247d 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -2,10 +2,13 @@ # Start vLLM-Ascend server in Docker for a given TP×DP configuration on 910C. # Usage: start_vllm_docker.sh # -# DSV4-Flash-specific: this launcher mounts host driver libs + all davinci dies -# explicitly via --device (NOT relying on Ascend Docker Runtime injection, -# which was unreliable on this host), and injects the DSV4-required vllm serve -# flags (tokenizer-mode, expert-parallel, quantization ascend, MTP, ...). +# DSV4-Flash-specific: mounts host driver libs + all 16 davinci dies explicitly +# via --device (NOT relying on Ascend Docker Runtime injection, which was +# unreliable here), and injects the DSV4-required vllm serve flags +# (tokenizer-mode, expert-parallel, quantization ascend, MTP, ...). +# +# Synced from glm52 fix (98cdb67): TP=16 device mount, configurable health +# timeout, driver mounts, container-name double-suffix fix. set -e TP="${1}" @@ -20,8 +23,9 @@ mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}" PORT="${VLLM_PORT:-30052}" -NAME="${CONTAINER_NAME:-${EXPERIMENT}_vllm_tp${TP}_dp${DP}}" -NAME="${NAME}_tp${TP}_dp${DP}" +# Fixed container name (no env override) to avoid the double-suffix bug where +# CONTAINER_NAME already carries _tpX_dpY and the launcher appends it again. +NAME="${CONTAINER_NAME}_tp${TP}_dp${DP}" PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" @@ -51,7 +55,7 @@ if [[ "$DP" -gt 1 ]]; then fi # ---- DSV4-Flash-specific args (REQUIRED; GLM-5.2 does not use these) ---- -# These were verified working in the manual bring-up on 910c.1. +# Verified working in the manual bring-up on 910c.1. SERVER_ARGS+=( --tokenizer-mode "$DSV4_TOKENIZER_MODE" --tool-call-parser "$DSV4_TOOL_CALL_PARSER" @@ -87,15 +91,8 @@ echo "Command: $SERVER_ARGS_STR" echo "Log: $LOG" # ---- Docker run ---- -# CRITICAL vs glm52 launcher: -# 1. Mount host driver libs (/usr/local/Ascend/driver/lib64, dcmi, hccn_tool, -# npu-smi, version.info, ascend_install.info, hccn.conf). Without these the -# container's torch_npu fails with "libascend_hal.so not found". -# 2. Mount all 16 dies explicitly via --device /dev/davinci0..15 plus the -# manager/devmm_svm/hisi_hdc control devices. Ascend Docker Runtime -# injection (ASCEND_VISIBLE_DEVICES only) was unreliable here. -# 3. --privileged + --shm-size 512g (DSV4 weight load needs large shm). -# 4. jemalloc preload path is the openEuler one (/usr/lib64/...). +# Mount host driver libs + all 16 dies explicitly (synced from glm52 fix). +# --privileged + --shm-size 512g for the 280GiB DSV4 weight load. DEVICE_ARGS=( --device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci2 --device /dev/davinci3 --device /dev/davinci4 --device /dev/davinci5 --device /dev/davinci6 --device /dev/davinci7 @@ -108,12 +105,13 @@ VOL_ARGS=( -v /usr/local/dcmi:/usr/local/dcmi -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi - -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ + -v /usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64 -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info -v /etc/ascend_install.info:/etc/ascend_install.info -v /etc/hccn.conf:/etc/hccn.conf -v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RUNTIME_BASE}/tmp:/tmp" + -v /mnt:/mnt ) ENV_ARGS=( @@ -157,9 +155,13 @@ echo $PID > "$PID_FILE" echo "PID: $PID" echo "Waiting for health on port ${PORT}..." -# DSV4 weight load (280 GiB) + graph compile takes much longer than GLM-5.2; -# raise the wait budget to 360 retries x 10s = 1 hour. -for i in $(seq 1 360); do +# Health-check budget. DSV4 weight load (280 GiB) + graph compile takes much +# longer than GLM-5.2; TP=16 compiles 16 graphs (~60min). Allow overrides. +# Synced from glm52 fix: 480 x 5s = 40min default. +HEALTH_MAX_RETRIES="${HEALTH_MAX_RETRIES:-480}" +HEALTH_RETRY_INTERVAL_S="${HEALTH_RETRY_INTERVAL_S:-5}" + +for i in $(seq 1 "$HEALTH_MAX_RETRIES"); do if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then echo "vLLM-Ascend server is ready at http://127.0.0.1:${PORT}" echo "Log: $LOG" @@ -170,10 +172,10 @@ for i in $(seq 1 360); do tail -200 "$LOG" exit 1 fi - echo "Waiting... ($i/360)" - sleep 10 + echo "Waiting... ($i/$HEALTH_MAX_RETRIES)" + sleep "$HEALTH_RETRY_INTERVAL_S" done -echo "ERROR: vLLM-Ascend server not healthy after 360 retries" +echo "ERROR: vLLM-Ascend server not healthy after $HEALTH_MAX_RETRIES retries" tail -200 "$LOG" exit 1