ea8302561e
- runner.py docker client 注入 TORCH_DEVICE_BACKEND_AUTOLOAD=0（bench 纯 HTTP client 不需要 NPU backend，跳过 torch_npu 加载失败） - dsv4/glm52 config.env: DOCKER_CLIENT_IMAGE 指向本地定制镜像 (local/vllm-ascend:0.23-a3-dsv4-sglang，内置 sglang 0.5.2 bench_serving)， USE_DOCKER_CLIENT=1，sskj.bench/run_bench.sh 的 docker client 分支可用 - run_bench.sh docker client 分支同步注入 AUTOLOAD=0 - glm52 profile 修正 DOCKER_IMAGE 为本机存在的 glm5.2-a3-openeuler
32 lines
2.9 KiB
Bash
32 lines
2.9 KiB
Bash
# GLM-5.2 w4a8c8 vLLM-Ascend TPxDP profile on a single 910C node
|
|
# (8x Ascend910 = 16 dies, 64GB HBM/die). GLM5.2-tuned image.
|
|
# Launch via BOOTSTRAP (base64) so the JSON-bearing args survive the image
|
|
# entrypoint; the container runs vllm serve in the foreground.
|
|
# Model-team only. Ops only run `python -m sskj.bench` against the served URL.
|
|
|
|
PLATFORM=910c
|
|
EXPERIMENT=glm52_910c_vllm_tp_dp_matrix
|
|
MODEL_NAME=GLM-5.2
|
|
ENGINE=vllm
|
|
RUNTIME=docker
|
|
DOCKER_IMAGE=quay.io/ascend/vllm-ascend:glm5.2-a3-openeuler
|
|
CONTAINER_NAME=vllm-ascend-glm52-910c_tp${TP}_dp${DP}
|
|
MODEL_PATH=/mnt/models/GLM-5.2-w4a8c8
|
|
SERVED_MODEL_NAME=glm-5.2
|
|
PORT=30050
|
|
HEALTH_PATH=/health
|
|
HEALTH_WAIT_S=2400
|
|
CONTAINER_PYTHON=/usr/local/python3.12.13/bin/python3
|
|
|
|
DEVICE_VARS="ASCEND_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
|
ENGINE_ENV="PYTORCH_NPU_ALLOC_CONF=expandable_segments:True PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 VLLM_ASCEND_ENABLE_FLASHCOMM1=1"
|
|
|
|
DOCKER_FLAGS="--privileged --ipc host --shm-size 512g --network host --ulimit memlock=-1 --device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci2 --device /dev/davinci3 --device /dev/davinci4 --device /dev/davinci5 --device /dev/davinci6 --device /dev/davinci7 --device /dev/davinci8 --device /dev/davinci9 --device /dev/davinci10 --device /dev/davinci11 --device /dev/davinci12 --device /dev/davinci13 --device /dev/davinci14 --device /dev/davinci15 --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc"
|
|
|
|
VOLUMES="/usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool /usr/local/bin/npu-smi:/usr/local/bin/npu-smi /usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64 /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info /etc/ascend_install.info:/etc/ascend_install.info /etc/hccn.conf:/etc/hccn.conf ${MODEL_PATH}:${MODEL_PATH}:ro /usr/local/dcmi:/usr/local/dcmi /mnt:/mnt"
|
|
|
|
LAUNCH_ARGS="${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --trust-remote-code --block-size 128 --tensor-parallel-size ${TP} --enable-expert-parallel --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.95} --max-model-len ${MAX_MODEL_LEN:-131072} --max-num-seqs ${MAX_NUM_SEQS:-256} --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS:-8192} --api-server-count 1 --host 0.0.0.0 --port ${PORT} --compilation-config '{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}' --additional-config '{\"enable_dsa_cp\": true,\"enable_sparse_sfa_c8\": false, \"enable_sparse_li_c8\": true,\"enable_balance_scheduling\": true,\"multistream_overlap_shared_expert\":true}'"
|
|
DP_FLAG="--data-parallel-size ${DP}"
|
|
|
|
BOOTSTRAP="for sh in /usr/local/Ascend/ascend-toolkit/set_env.sh /usr/local/Ascend/cann-9.0.1/share/info/ascendnpu-ir/bin/set_env.sh /usr/local/Ascend/cann-9.0.0/share/info/ascendnpu-ir/bin/set_env.sh /usr/local/Ascend/nnal/atb/set_env.sh; do [ -f \$sh ] && source \$sh && break; done; exec vllm serve ${LAUNCH_ARGS}"
|