98cdb67b66
sglang 0.5.2 bench_serving 输出与解析器不兼容,致 add16 c=32 崩溃: - 缺 total_throughput -> total_tps 恒0,无法检测吞吐增益 - 缺 p95_*(仅p99) -> TTFT SLO 失效 - gain previous<=0 返回 inf -> json.loads(inf) 崩溃 修复: 1. adaptive_concurrency.py: 缺失时从 ttfts/itls 数组补算 p95/p50; total_tps 回退 input+output throughput; gain 返回 Infinity 2. parse_backend.py: 同上补算逻辑; 补 from __future__ import annotations (py3.9 下 dict|None 语法无法 import) 3. start_vllm_docker.sh: --device davinci0~15 支持 TP=16; health 超时可配(默认480x5s=40min,TP=16编译16 graph约60min); 补驱动挂载+/mnt; 修容器名双后缀 4. run_adaptive_concurrency_add16.sh: --tokenizer 替代 --model; TORCH_DEVICE_BACKEND_AUTOLOAD=0; CONTAINER_PYTHON 路径; 导出 ENGINE_TP/DP 5. config.env: 固定 CONTAINER_NAME/DOCKER_IMAGE/GPU_MEM_UTIL 6. TP8_vs_TP16_report.md: TP=8 vs TP=16 手动测速对比报告 验证: TP=8 add16 c=16->c=32 不再崩溃; TP=16 编译完成变 healthy 推理正常
96 lines
4.2 KiB
Bash
96 lines
4.2 KiB
Bash
# TP×DP matrix experiment for GLM-5.2 on Ascend 910C (8 NPUs / 16 dies) using vLLM-Ascend.
|
||
# Tests vLLM with three parallel configurations:
|
||
# TP=2, DP=4 -> 2 dies per replica, 4 replicas
|
||
# TP=4, DP=2 -> 4 dies per replica, 2 replicas
|
||
# TP=8, DP=1 -> 8 dies, no data parallelism
|
||
#
|
||
# Platform: ascend_910c (see platforms/ascend_910c.env).
|
||
# Host: 910c.1 / NPU-NODE61, openEuler 22.03 SP4 aarch64, driver 25.5.2, CANN 9.0.0.
|
||
# Image: vllm-ascend; load the tarball from /mnt/models first (see envs/ASCEND_910C_ENV_SETUP.md).
|
||
|
||
EXPERIMENT="glm52_910c_vllm_tp_dp_matrix"
|
||
MODEL_NAME="GLM-5.2"
|
||
# GLM-5.2 ships two quantized variants on this host; w4a8c8 is the default.
|
||
# Switch to /mnt/models/GLM-5.2-w8a8 by overriding MODEL_PATH if needed.
|
||
MODEL_PATH="${MODEL_PATH:-/mnt/models/GLM-5.2-w4a8c8}"
|
||
SERVED_MODEL_NAME="glm-5.2"
|
||
|
||
VLLM_PORT="${VLLM_PORT:-30050}"
|
||
|
||
# Dedicated container name so this experiment never touches other 910c runs.
|
||
CONTAINER_NAME="vllm-ascend-glm52-910c"
|
||
|
||
# Python interpreter for the benchmark client inside the vllm-ascend container.
|
||
CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3"
|
||
|
||
# vllm-ascend image. Override with the exact tag after `docker load`-ing one of:
|
||
# /mnt/models/vllm-ascend-glm5.2-a3-openeuler.tar (GLM5.2-tuned, recommended)
|
||
# /mnt/models/vllm-ascend-v0.23.0rc1-a3-openeuler.tar (general v0.23)
|
||
USE_DOCKER="${USE_DOCKER:-1}"
|
||
DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-20260718-sglang}"
|
||
|
||
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving;
|
||
# reuse the vllm-ascend container itself for the client via `docker exec` (see
|
||
# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1
|
||
# with an external sglang image. Default off on 910c.
|
||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}"
|
||
|
||
# Device selection. ASCEND_VISIBLE_DEVICES selects NPU cards 0..7; the Ascend
|
||
# Docker Runtime (default runtime on this host) injects the matching dies.
|
||
export ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||
# Keep CUDA_VISIBLE_DEVICES for parity with the shared library; vllm-ascend
|
||
# ignores it on NPU but some helper code reads it.
|
||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||
|
||
# Runtime working directory for logs, pid files, and tmp.
|
||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||
|
||
# Parallel configurations to test. Format: "TP DP"
|
||
# Each Ascend910 card has 2 dies; TP addresses dies, so TP=8 uses 8 dies across
|
||
# 4 cards and leaves room for DP. TP=2/DP=4 and TP=4/DP=2 and TP=8/DP=1 all fit
|
||
# within 8 cards (16 dies). Override via PARALLEL_CONFIGS_STR="8,1".
|
||
if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then
|
||
declare -a PARALLEL_CONFIGS=()
|
||
for pair in $PARALLEL_CONFIGS_STR; do
|
||
PARALLEL_CONFIGS+=("${pair//,/ }")
|
||
done
|
||
else
|
||
declare -a PARALLEL_CONFIGS=(
|
||
"2 4"
|
||
"4 2"
|
||
"8 1"
|
||
)
|
||
fi
|
||
|
||
# vLLM-Ascend server settings for GLM-5.2 (w4a8c8).
|
||
# Notes:
|
||
# - KV cache dtype fp8 is supported on 910C; fall back to fp16 if the image rejects it.
|
||
# - block-size 128 matches Ascend page semantics (P99 of H20 uses 256; 910C favors 128).
|
||
# - MAX_MODEL_LEN: GLM-5.2 supports up to 128K context; cap at 131072.
|
||
# - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9 mirrors H20).
|
||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
|
||
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
|
||
BLOCK_SIZE="${BLOCK_SIZE:-128}"
|
||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
|
||
|
||
# vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh.
|
||
# attention backend for 910C: use the fused/atb attention path. Adjust per image.
|
||
VLLM_ASCEND_ATTENTION_BACKEND="${VLLM_ASCEND_ATTENTION_BACKEND:-atb}"
|
||
|
||
# Dataset used by sglang.bench_serving --dataset-name random.
|
||
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||
|
||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||
|
||
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||
|
||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||
|
||
DRY_RUN="${DRY_RUN:-0}"
|
||
GRID_LIMIT="${GRID_LIMIT:-0}"
|