fix(910c): bench client 可用化 - 本地 vllm-ascend-sglang 镜像 + torch_npu 自动加载禁用
- runner.py docker client 注入 TORCH_DEVICE_BACKEND_AUTOLOAD=0(bench 纯 HTTP client 不需要 NPU backend,跳过 torch_npu 加载失败) - dsv4/glm52 config.env: DOCKER_CLIENT_IMAGE 指向本地定制镜像 (local/vllm-ascend:0.23-a3-dsv4-sglang,内置 sglang 0.5.2 bench_serving), USE_DOCKER_CLIENT=1,sskj.bench/run_bench.sh 的 docker client 分支可用 - run_bench.sh docker client 分支同步注入 AUTOLOAD=0 - glm52 profile 修正 DOCKER_IMAGE 为本机存在的 glm5.2-a3-openeuler
This commit is contained in:
parent
6ba04325d3
commit
ea8302561e
@ -9,7 +9,7 @@ EXPERIMENT=glm52_910c_vllm_tp_dp_matrix
|
|||||||
MODEL_NAME=GLM-5.2
|
MODEL_NAME=GLM-5.2
|
||||||
ENGINE=vllm
|
ENGINE=vllm
|
||||||
RUNTIME=docker
|
RUNTIME=docker
|
||||||
DOCKER_IMAGE=local/vllm-ascend:0.23-a3-20260718-sglang
|
DOCKER_IMAGE=quay.io/ascend/vllm-ascend:glm5.2-a3-openeuler
|
||||||
CONTAINER_NAME=vllm-ascend-glm52-910c_tp${TP}_dp${DP}
|
CONTAINER_NAME=vllm-ascend-glm52-910c_tp${TP}_dp${DP}
|
||||||
MODEL_PATH=/mnt/models/GLM-5.2-w4a8c8
|
MODEL_PATH=/mnt/models/GLM-5.2-w4a8c8
|
||||||
SERVED_MODEL_NAME=glm-5.2
|
SERVED_MODEL_NAME=glm-5.2
|
||||||
|
|||||||
@ -31,12 +31,12 @@ CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3"
|
|||||||
USE_DOCKER="${USE_DOCKER:-1}"
|
USE_DOCKER="${USE_DOCKER:-1}"
|
||||||
DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}"
|
DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}"
|
||||||
|
|
||||||
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving;
|
# Benchmark client Docker image. The vllm-ascend images built on this host
|
||||||
# reuse the vllm-ascend container itself for the client via `docker exec` (see
|
# (local/vllm-ascend:0.23-a3-*-sglang, see envs/ASCEND_910C_ENV_SETUP.md §8.1)
|
||||||
# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1
|
# carry sglang 0.5.2 bench_serving, so the client runs as a standalone container
|
||||||
# with an external sglang image. Default off on 910c.
|
# (TORCH_DEVICE_BACKEND_AUTOLOAD=0 injected by the runner, no NPU needed).
|
||||||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}"
|
||||||
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}"
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
# Device selection. We mount all 8 cards (16 dies) via --device /dev/davinci0..15
|
# Device selection. We mount all 8 cards (16 dies) via --device /dev/davinci0..15
|
||||||
# in start_vllm_docker.sh. ASCEND_VISIBLE_DEVICES is kept for parity with the
|
# in start_vllm_docker.sh. ASCEND_VISIBLE_DEVICES is kept for parity with the
|
||||||
|
|||||||
@ -124,6 +124,7 @@ run_bench_serving() {
|
|||||||
-e HF_HUB_OFFLINE=1 \
|
-e HF_HUB_OFFLINE=1 \
|
||||||
-e TRANSFORMERS_OFFLINE=1 \
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
-e HF_DATASETS_OFFLINE=1 \
|
-e HF_DATASETS_OFFLINE=1 \
|
||||||
|
-e TORCH_DEVICE_BACKEND_AUTOLOAD=0 \
|
||||||
"${DOCKER_CLIENT_IMAGE}" \
|
"${DOCKER_CLIENT_IMAGE}" \
|
||||||
python -m sglang.bench_serving "$@"
|
python -m sglang.bench_serving "$@"
|
||||||
else
|
else
|
||||||
|
|||||||
@ -29,12 +29,12 @@ CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3"
|
|||||||
USE_DOCKER="${USE_DOCKER:-1}"
|
USE_DOCKER="${USE_DOCKER:-1}"
|
||||||
DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-20260718-sglang}"
|
DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-20260718-sglang}"
|
||||||
|
|
||||||
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving;
|
# Benchmark client Docker image. The vllm-ascend images built on this host
|
||||||
# reuse the vllm-ascend container itself for the client via `docker exec` (see
|
# (local/vllm-ascend:0.23-a3-*-sglang, see envs/ASCEND_910C_ENV_SETUP.md §8.1)
|
||||||
# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1
|
# carry sglang 0.5.2 bench_serving, so the client runs as a standalone container
|
||||||
# with an external sglang image. Default off on 910c.
|
# (TORCH_DEVICE_BACKEND_AUTOLOAD=0 injected by the runner, no NPU needed).
|
||||||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}"
|
||||||
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}"
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
# Device selection. ASCEND_VISIBLE_DEVICES selects NPU cards 0..7; the Ascend
|
# Device selection. ASCEND_VISIBLE_DEVICES selects NPU cards 0..7; the Ascend
|
||||||
# Docker Runtime (default runtime on this host) injects the matching dies.
|
# Docker Runtime (default runtime on this host) injects the matching dies.
|
||||||
|
|||||||
@ -115,6 +115,7 @@ run_bench_serving() {
|
|||||||
-e HF_HUB_OFFLINE=1 \
|
-e HF_HUB_OFFLINE=1 \
|
||||||
-e TRANSFORMERS_OFFLINE=1 \
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
-e HF_DATASETS_OFFLINE=1 \
|
-e HF_DATASETS_OFFLINE=1 \
|
||||||
|
-e TORCH_DEVICE_BACKEND_AUTOLOAD=0 \
|
||||||
"${DOCKER_CLIENT_IMAGE}" \
|
"${DOCKER_CLIENT_IMAGE}" \
|
||||||
python -m sglang.bench_serving "$@"
|
python -m sglang.bench_serving "$@"
|
||||||
else
|
else
|
||||||
|
|||||||
@ -113,7 +113,16 @@ def run_scenario(scenario: dict[str, Any], options: BenchClientOptions) -> int:
|
|||||||
cmd += ["-v", f"{options.dataset_path}:{options.dataset_path}:ro"]
|
cmd += ["-v", f"{options.dataset_path}:{options.dataset_path}:ro"]
|
||||||
if options.output_file:
|
if options.output_file:
|
||||||
cmd += ["-v", f"{options.output_file.parent}:{options.output_file.parent}"]
|
cmd += ["-v", f"{options.output_file.parent}:{options.output_file.parent}"]
|
||||||
cmd += ["-e", "HF_HUB_OFFLINE=1", "-e", "TRANSFORMERS_OFFLINE=1", "-e", "HF_DATASETS_OFFLINE=1"]
|
cmd += [
|
||||||
|
"-e",
|
||||||
|
"HF_HUB_OFFLINE=1",
|
||||||
|
"-e",
|
||||||
|
"TRANSFORMERS_OFFLINE=1",
|
||||||
|
"-e",
|
||||||
|
"HF_DATASETS_OFFLINE=1",
|
||||||
|
"-e",
|
||||||
|
"TORCH_DEVICE_BACKEND_AUTOLOAD=0",
|
||||||
|
]
|
||||||
cmd += [
|
cmd += [
|
||||||
options.client_image,
|
options.client_image,
|
||||||
"python",
|
"python",
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user