fix(dsv4): sync glm52 add16 fixes (sglang 0.5.2 client + health timeout)

Sync the glm52 add16 fixes (98cdb67) into the DSV4 experiment so the
adaptive concurrency search can actually run end-to-end.

sglang client (the main blocker):
- Built local/vllm-ascend:0.23-a3-dsv4-sglang image: sglang 0.5.2 (not
  0.5.16 -- 0.5.16 deprecates bench_serving and the glm52 parser fix
  targets the 0.5.2 output format) + minimal deps (ipython/traitlets/
  stack_data/executing/asttokens/pure_eval/prompt_toolkit/wcwidth) via
  --no-deps, so the vllm env is untouched.
- Verified: python -m sglang.bench_serving --help works in the image.
- config.env DOCKER_IMAGE -> local/vllm-ascend:0.23-a3-dsv4-sglang.

config.env (sync glm52 98cdb67):
- CONTAINER_NAME: drop ${...:-} override -> fixed value (avoids the
  double-suffix bug where CONTAINER_NAME already carries _tpX_dpY).
- CONTAINER_PYTHON: /usr/local/bin/python (does not exist) ->
  /usr/local/python3.12.13/bin/python3 (matches glm52 fix).

run_adaptive_concurrency_add16.sh (sync glm52 98cdb67):
- --model $SERVED_MODEL_NAME -> --tokenizer $MODEL_PATH (bench_serving
  0.5.2 wants the tokenizer path).
- docker exec env: add TORCH_DEVICE_BACKEND_AUTOLOAD=0 so the client
  does not try to autoload torch_npu.
- export ENGINE_TP/ENGINE_DP in engine_start_server + export line;
  container_name uses ${ENGINE_TP:-${tp}} (the bench runs in a subshell
  where tp/dp are not in scope).

start_vllm_docker.sh (sync glm52 98cdb67):
- Health timeout configurable via HEALTH_MAX_RETRIES /
  HEALTH_RETRY_INTERVAL_S (default 480x5s=40min; TP=16 compiles 16
  graphs ~60min, old hardcoded 360x10s was too rigid).
- Container name: drop the double-suffix (CONTAINER_NAME no longer
  re-overridden before appending _tpX_dpY).
- Mount /mnt (bench client reads dataset from there).

experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env

@@ -20,16 +20,16 @@ SERVED_MODEL_NAME="dsv4"
 VLLM_PORT="${VLLM_PORT:-30052}"
 
 # Dedicated container name so this experiment never touches other 910c runs.
-CONTAINER_NAME="${CONTAINER_NAME:-vllm-ascend-dsv4-910c}"
+CONTAINER_NAME="vllm-ascend-dsv4-910c"
 
 # Python interpreter for the benchmark client inside the vllm-ascend container.
-CONTAINER_PYTHON="${CONTAINER_PYTHON:-/usr/local/bin/python}"
+CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3"
 
 # vllm-ascend image. Use the general v0.23 A3 image for DSV4 (the GLM5.2-tuned
 # variant carries GLM-specific patches and is NOT compatible with DSV4).
 # Verified present locally: quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler
 USE_DOCKER="${USE_DOCKER:-1}"
-DOCKER_IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}"
+DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}"
 
 # Benchmark client Docker image. vLLM's image does not include sglang.bench_serving;
 # reuse the vllm-ascend container itself for the client via `docker exec` (see

experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -97,6 +97,8 @@ engine_start_server() {
   local tp="$1"
   local dp="$2"
   local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  export ENGINE_TP="$tp"
+  export ENGINE_DP="$dp"
   log "starting vllm-ascend server tp=${tp} dp=${dp}"
   bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
   if ! engine_is_healthy; then
@@ -117,6 +119,8 @@ engine_detect_oom() {
   local dp="$3"
   local pattern='out of memory|OutOfMemory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory|NPU out of memory|acl.*memory|HBM'
   local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  export ENGINE_TP="$tp"
+  export ENGINE_DP="$dp"
   local -a logs=("$detail_log" "$outer_log")
   if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
     logs+=("$ACTIVE_ENGINE_SERVER_LOG")
@@ -130,7 +134,7 @@ engine_run_bench() {
   local concurrency="$3"
   local num_prompts="$4"
   local output_file="$5"
-  local container_name="${CONTAINER_NAME}_tp${tp}_dp${dp}"
+  local container_name="${CONTAINER_NAME}_tp${ENGINE_TP:-${tp}}_dp${ENGINE_DP:-${dp}}"
   local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")"
   if [[ "$output_file" == "/dev/null" ]]; then
     container_output="/dev/null"
@@ -143,7 +147,7 @@ engine_run_bench() {
     --backend vllm
     --host 127.0.0.1
     --port "$ENGINE_PORT"
-    --model "$SERVED_MODEL_NAME"
+    --tokenizer "$MODEL_PATH"
     --dataset-name "$BENCH_DATASET_NAME"
     --random-input-len "$isl"
     --random-output-len "$osl"
@@ -170,7 +174,7 @@ engine_run_bench() {
   # host port. See envs/ASCEND_910C_ENV_SETUP.md.
   docker exec "$container_name" mkdir -p /tmp/bench_outputs 2>/dev/null || true
   docker exec "$container_name" \
-    env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \
+    env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 TORCH_DEVICE_BACKEND_AUTOLOAD=0 \
     "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $?
 
   if [[ "$output_file" != "/dev/null" ]]; then
@@ -179,7 +183,7 @@ engine_run_bench() {
 }
 
 export -f engine_run_bench
-export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE SERVED_MODEL_NAME
+export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE SERVED_MODEL_NAME ENGINE_TP ENGINE_DP
 export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
 
 export SEARCH_START_CONCURRENCY=16

experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh

@@ -2,10 +2,13 @@
 # Start vLLM-Ascend server in Docker for a given TP×DP configuration on 910C.
 # Usage: start_vllm_docker.sh <TP> <DP>
 #
-# DSV4-Flash-specific: this launcher mounts host driver libs + all davinci dies
+# DSV4-Flash-specific: mounts host driver libs + all 16 davinci dies explicitly
-# explicitly via --device (NOT relying on Ascend Docker Runtime injection,
+# via --device (NOT relying on Ascend Docker Runtime injection, which was
-# which was unreliable on this host), and injects the DSV4-required vllm serve
+# unreliable here), and injects the DSV4-required vllm serve flags
-# flags (tokenizer-mode, expert-parallel, quantization ascend, MTP, ...).
+# (tokenizer-mode, expert-parallel, quantization ascend, MTP, ...).
+#
+# Synced from glm52 fix (98cdb67): TP=16 device mount, configurable health
+# timeout, driver mounts, container-name double-suffix fix.
 set -e
 
 TP="${1}"
@@ -20,8 +23,9 @@ mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
 
 IMAGE="${DOCKER_IMAGE:-quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler}"
 PORT="${VLLM_PORT:-30052}"
-NAME="${CONTAINER_NAME:-${EXPERIMENT}_vllm_tp${TP}_dp${DP}}"
+# Fixed container name (no env override) to avoid the double-suffix bug where
-NAME="${NAME}_tp${TP}_dp${DP}"
+# CONTAINER_NAME already carries _tpX_dpY and the launcher appends it again.
+NAME="${CONTAINER_NAME}_tp${TP}_dp${DP}"
 PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
 
 LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
@@ -51,7 +55,7 @@ if [[ "$DP" -gt 1 ]]; then
 fi
 
 # ---- DSV4-Flash-specific args (REQUIRED; GLM-5.2 does not use these) ----
-# These were verified working in the manual bring-up on 910c.1.
+# Verified working in the manual bring-up on 910c.1.
 SERVER_ARGS+=(
   --tokenizer-mode "$DSV4_TOKENIZER_MODE"
   --tool-call-parser "$DSV4_TOOL_CALL_PARSER"
@@ -87,15 +91,8 @@ echo "Command: $SERVER_ARGS_STR"
 echo "Log: $LOG"
 
 # ---- Docker run ----
-# CRITICAL vs glm52 launcher:
+# Mount host driver libs + all 16 dies explicitly (synced from glm52 fix).
-#   1. Mount host driver libs (/usr/local/Ascend/driver/lib64, dcmi, hccn_tool,
+# --privileged + --shm-size 512g for the 280GiB DSV4 weight load.
-#      npu-smi, version.info, ascend_install.info, hccn.conf). Without these the
-#      container's torch_npu fails with "libascend_hal.so not found".
-#   2. Mount all 16 dies explicitly via --device /dev/davinci0..15 plus the
-#      manager/devmm_svm/hisi_hdc control devices. Ascend Docker Runtime
-#      injection (ASCEND_VISIBLE_DEVICES only) was unreliable here.
-#   3. --privileged + --shm-size 512g (DSV4 weight load needs large shm).
-#   4. jemalloc preload path is the openEuler one (/usr/lib64/...).
 DEVICE_ARGS=(
   --device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci2 --device /dev/davinci3
   --device /dev/davinci4 --device /dev/davinci5 --device /dev/davinci6 --device /dev/davinci7
@@ -108,12 +105,13 @@ VOL_ARGS=(
   -v /usr/local/dcmi:/usr/local/dcmi
   -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool
   -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
-  -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/
+  -v /usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64
   -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info
   -v /etc/ascend_install.info:/etc/ascend_install.info
   -v /etc/hccn.conf:/etc/hccn.conf
   -v "${MODEL_PATH}:${MODEL_PATH}:ro"
   -v "${RUNTIME_BASE}/tmp:/tmp"
+  -v /mnt:/mnt
 )
 
 ENV_ARGS=(
@@ -157,9 +155,13 @@ echo $PID > "$PID_FILE"
 echo "PID: $PID"
 echo "Waiting for health on port ${PORT}..."
 
-# DSV4 weight load (280 GiB) + graph compile takes much longer than GLM-5.2;
+# Health-check budget. DSV4 weight load (280 GiB) + graph compile takes much
-# raise the wait budget to 360 retries x 10s = 1 hour.
+# longer than GLM-5.2; TP=16 compiles 16 graphs (~60min). Allow overrides.
-for i in $(seq 1 360); do
+# Synced from glm52 fix: 480 x 5s = 40min default.
+HEALTH_MAX_RETRIES="${HEALTH_MAX_RETRIES:-480}"
+HEALTH_RETRY_INTERVAL_S="${HEALTH_RETRY_INTERVAL_S:-5}"
+
+for i in $(seq 1 "$HEALTH_MAX_RETRIES"); do
   if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
     echo "vLLM-Ascend server is ready at http://127.0.0.1:${PORT}"
     echo "Log: $LOG"
@@ -170,10 +172,10 @@ for i in $(seq 1 360); do
     tail -200 "$LOG"
     exit 1
   fi
-  echo "Waiting... ($i/360)"
+  echo "Waiting... ($i/$HEALTH_MAX_RETRIES)"
-  sleep 10
+  sleep "$HEALTH_RETRY_INTERVAL_S"
 done
 
-echo "ERROR: vLLM-Ascend server not healthy after 360 retries"
+echo "ERROR: vLLM-Ascend server not healthy after $HEALTH_MAX_RETRIES retries"
 tail -200 "$LOG"
 exit 1