From 4197e2738d60ad0343d92ae818a041b316f3fba3 Mon Sep 17 00:00:00 2001 From: shishi Date: Tue, 28 Jul 2026 16:36:53 +0800 Subject: [PATCH] feat(dsv4): make DSV4-Flash 910C experiment runnable (verified TP4/DP2) The dsv4_910c_vllm_tp_dp_matrix experiment was a placeholder (wrong MODEL_PATH, glm52 image default, no DSV4 serve flags, no driver mounts). Bring it to a working state, validated end-to-end on 910c.1. config.env: - MODEL_PATH: /mnt/models/DeepSeek-V4-Flash -> .../DeepSeek-V4-Flash-w8a8-mtp (weights downloaded from ModelScope Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp, ~280 GiB, 70 shards, verified present and chowned to shishi) - SERVED_MODEL_NAME: deepseek-v4-flash -> dsv4 (matches vllm-ascend tutorial) - DOCKER_IMAGE: vllm-ascend:v0.23.0rc1-a3-openeuler -> quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler (full tag, present locally; the glm5.2-a3 image carries GLM-specific patches and is NOT DSV4-compatible) - Add DSV4_* serve-flag vars: tokenizer-mode/tool-call-parser/reasoning-parser deepseek_v4, quantization ascend, expert-parallel, async-scheduling, MTP speculative-config, compilation-config, additional-config, multithread model-loader (128 threads for the 280GiB load) - Rewrite the OOM-boundary comment to reflect actual w8a8 weight size start_vllm_docker.sh (the main pitfalls vs the glm52 launcher): - Inject the DSV4_* serve flags (GLM-5.2 needs none of them); without them vllm rejects the model / lacks MTP - Mount host driver libs (driver/lib64, dcmi, hccn_tool, npu-smi, version.info, ascend_install.info, hccn.conf) -- otherwise the container torch_npu fails with libascend_hal.so not found - Mount all 16 dies via --device /dev/davinci0..15 + davinci_manager/ devmm_svm/hisi_hdc instead of relying on Ascend Docker Runtime injection (ASCEND_VISIBLE_DEVICES-only), which was unreliable on this host - --privileged --shm-size 512g for the 280GiB weight load - LD_PRELOAD the openEuler jemalloc path /usr/lib64/libjemalloc.so.2 (the glm52 ubuntu path /usr/lib/aarch64-linux-gnu/... does not exist here) - Raise health-wait budget 240x5s -> 360x10s (DSV4 load+compile ~8min) - DRY_RUN mode for command preview without launching Verified: start_vllm_docker.sh 4 2 brings the server up on port 30052 in ~8 min (130s weight load per die, 29s compile, 187s engine init); chat completion returns correctly, system_fingerprint vllm-0.23.0-tp4-dp2-ep. --- .../dsv4_910c_vllm_tp_dp_matrix/config.env | 84 ++++++++---- .../start_vllm_docker.sh | 120 ++++++++++++++---- 2 files changed, 158 insertions(+), 46 deletions(-) diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env index 2b586c4..52ac44c 100644 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env @@ -8,17 +8,14 @@ # Platform: ascend_910c (see platforms/ascend_910c.env). # Host: 910c.1 / NPU-NODE61, openEuler 22.03 SP4 aarch64, driver 25.5.2, CANN 9.0.0. # -# ⚠️ PLACEHOLDER: DeepSeek-V4-Flash weights are NOT yet present on this host -# (only GLM-5.2-{w4a8c8,w8a8} under /mnt/models). Download DSV4-Flash first, -# then fix MODEL_PATH below. Smoke run will fail until the model exists. -# Recommended host path: /mnt/models/DeepSeek-V4-Flash (FP8) -# /mnt/models/DeepSeek-V4-Flash-INT8 (INT8) +# DeepSeek-V4-Flash w8a8-mtp weights are present at /mnt/models/DeepSeek-V4-Flash-w8a8-mtp +# (downloaded from ModelScope Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp, ~280 GiB, 70 shards). EXPERIMENT="dsv4_910c_vllm_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -# TODO: point this at the real DSV4-Flash directory once downloaded. -MODEL_PATH="${MODEL_PATH:-/mnt/models/DeepSeek-V4-Flash}" -SERVED_MODEL_NAME="deepseek-v4-flash" +# Real DSV4-Flash w8a8-mtp directory (verified present & chown'd to shishi). +MODEL_PATH="${MODEL_PATH:-/mnt/models/DeepSeek-V4-Flash-w8a8-mtp}" +SERVED_MODEL_NAME="dsv4" VLLM_PORT="${VLLM_PORT:-30052}" @@ -28,28 +25,36 @@ CONTAINER_NAME="${CONTAINER_NAME:-vllm-ascend-dsv4-910c}" # Python interpreter for the benchmark client inside the vllm-ascend container. CONTAINER_PYTHON="${CONTAINER_PYTHON:-/usr/local/bin/python}" -# vllm-ascend image. Use the general v0.23 image for DSV4 (the GLM5.2-tuned -# variant may carry GLM-specific patches). Load from: -# /mnt/models/vllm-ascend-v0.23.0rc1-a3-openeuler.tar +# vllm-ascend image. Use the general v0.23 A3 image for DSV4 (the GLM5.2-tuned +# variant carries GLM-specific patches and is NOT compatible with DSV4). +# Verified present locally: quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler USE_DOCKER="${USE_DOCKER:-1}" -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-ascend:v0.23.0rc1-a3-openeuler}" +DOCKER_IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}" +# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving; +# reuse the vllm-ascend container itself for the client via `docker exec` (see +# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1 +# with an external sglang image. Default off on 910c. DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}" +# Device selection. We mount all 8 cards (16 dies) via --device /dev/davinci0..15 +# in start_vllm_docker.sh. ASCEND_VISIBLE_DEVICES is kept for parity with the +# shared library but the explicit --device flags are the authoritative path on +# this host (Ascend Docker Runtime injection was unreliable here). export ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +# Runtime working directory for logs, pid files, and tmp. RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" # Parallel configurations to test. Format: "TP DP" -# DSV4-Flash FP8 routed-expert weights ~264 GiB total. Per-die load = 264/TP GiB. -# TP=2 -> 132 GiB/die > 64 GiB HBM ❌ OOM expected (kept but will fail) -# TP=4 -> 66 GiB/die > 64 GiB HBM ❌ borderline OOM (KV cache leaves no room) -# TP=8 -> 33 GiB/die < 64 GiB HBM ✅ fits with room for KV cache -# TP=2/DP=4 and TP=4/DP=2 are likely infeasible for FP8 on 64GB dies; they are -# kept here so the smoke run records the OOM boundary explicitly. Switch to -# INT8 weights (MODEL_PATH=...-INT8, ~132 GiB total) to make TP=4 viable. +# DSV4-Flash w8a8 routed-expert weights ~280 GiB total. With expert-parallel the +# weight load is sharded across TP dies, but w8a8 still leaves a heavy per-die +# footprint. TP*DP must equal 16 (8 cards x 2 dies). +# TP=2, DP=4 -> 4 dies/replica x 4 replicas (smallest TP, most replicas) +# TP=4, DP=2 -> 4 dies/replica x 2 replicas +# TP=8, DP=1 -> 8 dies/replica x 1 replica (largest TP, max weight sharding) if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then declare -a PARALLEL_CONFIGS=() for pair in $PARALLEL_CONFIGS_STR; do @@ -63,19 +68,54 @@ else ) fi -# vLLM-Ascend server settings for DSV4-Flash. +# vLLM-Ascend server settings for DSV4-Flash (w8a8-mtp). +# Notes: +# - KV cache dtype fp8 is supported on 910C; fall back to fp16 if the image rejects it. +# - block-size 128 matches Ascend page semantics (910C favors 128). +# - MAX_MODEL_LEN: DSV4-Flash supports up to 1M context; cap at 131072 for the +# matrix sweep (extend to 1M via matrix.json once TP=8 is verified). +# - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9). GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" BLOCK_SIZE="${BLOCK_SIZE:-128}" -# DSV4-Flash supports up to 1M context, but 64GB HBM/die caps realistic ISL. -# Start at 128K; extend via matrix.json once TP=8 is verified. MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +# vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh. +# attention backend for 910C: use the fused/atb attention path. Adjust per image. VLLM_ASCEND_ATTENTION_BACKEND="${VLLM_ASCEND_ATTENTION_BACKEND:-atb}" +# DSV4-Flash-specific server args (passed through to vllm serve in +# start_vllm_docker.sh). These are REQUIRED for DSV4 -- GLM-5.2 does not need them. +# --tokenizer-mode deepseek_v4 : DSV4 custom tokenizer +# --tool-call-parser deepseek_v4 : DSV4 tool-call parser +# --enable-auto-tool-choice : enable automatic tool choice +# --reasoning-parser deepseek_v4 : DSV4 reasoning parser +# --enable-expert-parallel : shard 256 experts across EP ranks +# --quantization ascend : use modelslim w8a8 quantization path +# --safetensors-load-strategy prefetch: prefetch shards for faster load +# --async-scheduling : overlap CPU scheduling with NPU compute +# MTP speculative decoding via SPEC_CONFIG +DSV4_TOKENIZER_MODE="${DSV4_TOKENIZER_MODE:-deepseek_v4}" +DSV4_TOOL_CALL_PARSER="${DSV4_TOOL_CALL_PARSER:-deepseek_v4}" +DSV4_REASONING_PARSER="${DSV4_REASONING_PARSER:-deepseek_v4}" +DSV4_QUANTIZATION="${DSV4_QUANTIZATION:-ascend}" +DSV4_SAFETENSORS_LOAD_STRATEGY="${DSV4_SAFETENSORS_LOAD_STRATEGY:-prefetch}" +# MTP speculative config (1 speculative token). JSON string, kept single-quoted +# in the launcher to avoid shell mangling. +DSV4_SPEC_CONFIG="${DSV4_SPEC_CONFIG:-{\"num_speculative_tokens\": 1, \"method\": \"mtp\", \"enforce_eager\": true}}" +DSV4_COMPILATION_CONFIG="${DSV4_COMPILATION_CONFIG:-{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}}" +DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"enable_dsa_cp\": true,\"multistream_overlap_shared_expert\":true}}" +DSV4_ENABLE_EXPERT_PARALLEL="${DSV4_ENABLE_EXPERT_PARALLEL:-1}" +DSV4_ENABLE_ASYNC_SCHEDULING="${DSV4_ENABLE_ASYNC_SCHEDULING:-1}" + +# Model-loader extra config (multithread load, 128 threads) to speed up 280GiB load. +DSV4_MODEL_LOADER_EXTRA_CONFIG="${DSV4_MODEL_LOADER_EXTRA_CONFIG:-{\"enable_multithread_load\": \"true\", \"num_threads\": 128}}" + +# Dataset used by sglang.bench_serving --dataset-name random. DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" +# Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index 2553617..c63d3a2 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -2,9 +2,10 @@ # Start vLLM-Ascend server in Docker for a given TP×DP configuration on 910C. # Usage: start_vllm_docker.sh # -# Relies on the Ascend Docker Runtime being the default docker runtime on this -# host (see /etc/docker/daemon.json). NPU dies are injected via the -# ASCEND_VISIBLE_DEVICES env var; no --gpus / --device flags are needed. +# DSV4-Flash-specific: this launcher mounts host driver libs + all davinci dies +# explicitly via --device (NOT relying on Ascend Docker Runtime injection, +# which was unreliable on this host), and injects the DSV4-required vllm serve +# flags (tokenizer-mode, expert-parallel, quantization ascend, MTP, ...). set -e TP="${1}" @@ -17,8 +18,8 @@ source "${SCRIPT_DIR}/config.env" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" -IMAGE="${DOCKER_IMAGE:-vllm-ascend:glm5.2-a3-openeuler}" -PORT="${VLLM_PORT:-30050}" +IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}" +PORT="${VLLM_PORT:-30052}" NAME="${CONTAINER_NAME:-${EXPERIMENT}_vllm_tp${TP}_dp${DP}}" NAME="${NAME}_tp${TP}_dp${DP}" PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" @@ -29,9 +30,8 @@ rm -f "$PID_FILE" # Clean up any stale container with the same name. docker rm -f "$NAME" >/dev/null 2>&1 || true -# vLLM-Ascend launch args. Differences vs NVIDIA vLLM: -# - no --no-enable-flashinfer-autotune (Ascend uses its own attention path) -# - --kv-cache-dtype may need to be fp16 if the image rejects fp8 on 910C +# ---- vLLM-Ascend launch args ---- +# Base args common to GLM-5.2 and DSV4-Flash. SERVER_ARGS=( "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" @@ -50,6 +50,31 @@ if [[ "$DP" -gt 1 ]]; then SERVER_ARGS+=(--data-parallel-size "$DP") fi +# ---- DSV4-Flash-specific args (REQUIRED; GLM-5.2 does not use these) ---- +# These were verified working in the manual bring-up on 910c.1. +SERVER_ARGS+=( + --tokenizer-mode "$DSV4_TOKENIZER_MODE" + --tool-call-parser "$DSV4_TOOL_CALL_PARSER" + --enable-auto-tool-choice + --reasoning-parser "$DSV4_REASONING_PARSER" + --quantization "$DSV4_QUANTIZATION" + --safetensors-load-strategy "$DSV4_SAFETENSORS_LOAD_STRATEGY" +) + +if [[ "${DSV4_ENABLE_EXPERT_PARALLEL:-1}" == "1" ]]; then + SERVER_ARGS+=(--enable-expert-parallel) +fi + +if [[ "${DSV4_ENABLE_ASYNC_SCHEDULING:-1}" == "1" ]]; then + SERVER_ARGS+=(--async-scheduling) +fi + +# JSON-bearing args. Quoted as single strings so vllm parses each as one argv. +SERVER_ARGS+=(--model-loader-extra-config "$DSV4_MODEL_LOADER_EXTRA_CONFIG") +SERVER_ARGS+=(--speculative-config "$DSV4_SPEC_CONFIG") +SERVER_ARGS+=(--compilation-config "$DSV4_COMPILATION_CONFIG") +SERVER_ARGS+=(--additional-config "$DSV4_ADDITIONAL_CONFIG") + SERVER_ARGS_STR="vllm serve ${SERVER_ARGS[*]}" echo "=== Starting vLLM-Ascend server in Docker (TP=${TP}, DP=${DP}) ===" @@ -61,23 +86,68 @@ echo "ASCEND_VISIBLE_DEVICES: ${ASCEND_VISIBLE_DEVICES}" echo "Command: $SERVER_ARGS_STR" echo "Log: $LOG" -# Run docker in the foreground; nohup backgrounds it and the host PID lets the -# adaptive search stop the container by killing the process (the container has -# --rm so it self-cleans). The Ascend Docker Runtime is the default runtime, so -# no --runtime flag is required. +# ---- Docker run ---- +# CRITICAL vs glm52 launcher: +# 1. Mount host driver libs (/usr/local/Ascend/driver/lib64, dcmi, hccn_tool, +# npu-smi, version.info, ascend_install.info, hccn.conf). Without these the +# container's torch_npu fails with "libascend_hal.so not found". +# 2. Mount all 16 dies explicitly via --device /dev/davinci0..15 plus the +# manager/devmm_svm/hisi_hdc control devices. Ascend Docker Runtime +# injection (ASCEND_VISIBLE_DEVICES only) was unreliable here. +# 3. --privileged + --shm-size 512g (DSV4 weight load needs large shm). +# 4. jemalloc preload path is the openEuler one (/usr/lib64/...). +DEVICE_ARGS=( + --device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci2 --device /dev/davinci3 + --device /dev/davinci4 --device /dev/davinci5 --device /dev/davinci6 --device /dev/davinci7 + --device /dev/davinci8 --device /dev/davinci9 --device /dev/davinci10 --device /dev/davinci11 + --device /dev/davinci12 --device /dev/davinci13 --device /dev/davinci14 --device /dev/davinci15 + --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc +) + +VOL_ARGS=( + -v /usr/local/dcmi:/usr/local/dcmi + -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi + -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ + -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info + -v /etc/ascend_install.info:/etc/ascend_install.info + -v /etc/hccn.conf:/etc/hccn.conf + -v "${MODEL_PATH}:${MODEL_PATH}:ro" + -v "${RUNTIME_BASE}/tmp:/tmp" +) + +ENV_ARGS=( + -e ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES}" + -e PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + -e PYTHONUNBUFFERED=1 + -e HF_HUB_OFFLINE=1 + -e TRANSFORMERS_OFFLINE=1 + -e OMP_PROC_BIND=false + -e OMP_NUM_THREADS=10 + -e HCCL_BUFFSIZE=1024 + -e VLLM_ASCEND_ENABLE_FLASHCOMM1=1 + -e TASK_QUEUE_ENABLE=1 + -e "HCCL_OP_EXPANSION_MODE=AIV" + -e "LD_PRELOAD=/usr/lib64/libjemalloc.so.2" +) + +if [[ "${DRY_RUN:-0}" == "1" ]]; then + echo "[DRY-RUN] docker run --rm --name \"$NAME\" --privileged --ipc host --shm-size 512g \\" + echo " --network host ${DEVICE_ARGS[*]} ${VOL_ARGS[*]} ${ENV_ARGS[*]} \\" + echo " \"$IMAGE\" $SERVER_ARGS_STR" + exit 0 +fi + nohup docker run --rm \ --name "$NAME" \ + --privileged \ --ipc host \ - --shm-size 16g \ + --shm-size 512g \ --network host \ --ulimit memlock=-1 \ - -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ - -v "${RUNTIME_BASE}/tmp:/tmp" \ - -e ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES}" \ - -e PYTORCH_NPU_ALLOC_CONF=expandable_segments:True \ - -e PYTHONUNBUFFERED=1 \ - -e HF_HUB_OFFLINE=1 \ - -e TRANSFORMERS_OFFLINE=1 \ + "${DEVICE_ARGS[@]}" \ + "${VOL_ARGS[@]}" \ + "${ENV_ARGS[@]}" \ "$IMAGE" \ vllm serve "${SERVER_ARGS[@]}" \ > "$LOG" 2>&1 & @@ -87,7 +157,9 @@ echo $PID > "$PID_FILE" echo "PID: $PID" echo "Waiting for health on port ${PORT}..." -for i in $(seq 1 240); do +# DSV4 weight load (280 GiB) + graph compile takes much longer than GLM-5.2; +# raise the wait budget to 360 retries x 10s = 1 hour. +for i in $(seq 1 360); do if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then echo "vLLM-Ascend server is ready at http://127.0.0.1:${PORT}" echo "Log: $LOG" @@ -98,10 +170,10 @@ for i in $(seq 1 240); do tail -200 "$LOG" exit 1 fi - echo "Waiting... ($i/240)" - sleep 5 + echo "Waiting... ($i/360)" + sleep 10 done -echo "ERROR: vLLM-Ascend server not healthy after 240 retries" +echo "ERROR: vLLM-Ascend server not healthy after 360 retries" tail -200 "$LOG" exit 1