diff --git a/deploy/profiles/910c/glm52_910c_vllm_tp_dp_matrix.env b/deploy/profiles/910c/glm52_910c_vllm_tp_dp_matrix.env index 99f2195..7582abf 100644 --- a/deploy/profiles/910c/glm52_910c_vllm_tp_dp_matrix.env +++ b/deploy/profiles/910c/glm52_910c_vllm_tp_dp_matrix.env @@ -9,7 +9,7 @@ EXPERIMENT=glm52_910c_vllm_tp_dp_matrix MODEL_NAME=GLM-5.2 ENGINE=vllm RUNTIME=docker -DOCKER_IMAGE=local/vllm-ascend:0.23-a3-20260718-sglang +DOCKER_IMAGE=quay.io/ascend/vllm-ascend:glm5.2-a3-openeuler CONTAINER_NAME=vllm-ascend-glm52-910c_tp${TP}_dp${DP} MODEL_PATH=/mnt/models/GLM-5.2-w4a8c8 SERVED_MODEL_NAME=glm-5.2 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env index 73d8868..7e02106 100644 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env @@ -31,12 +31,12 @@ CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3" USE_DOCKER="${USE_DOCKER:-1}" DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}" -# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving; -# reuse the vllm-ascend container itself for the client via `docker exec` (see -# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1 -# with an external sglang image. Default off on 910c. -DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" -USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}" +# Benchmark client Docker image. The vllm-ascend images built on this host +# (local/vllm-ascend:0.23-a3-*-sglang, see envs/ASCEND_910C_ENV_SETUP.md §8.1) +# carry sglang 0.5.2 bench_serving, so the client runs as a standalone container +# (TORCH_DEVICE_BACKEND_AUTOLOAD=0 injected by the runner, no NPU needed). +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}" +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" # Device selection. We mount all 8 cards (16 dies) via --device /dev/davinci0..15 # in start_vllm_docker.sh. ASCEND_VISIBLE_DEVICES is kept for parity with the diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_bench.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_bench.sh index b7fd5c2..68dff58 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_bench.sh @@ -124,6 +124,7 @@ run_bench_serving() { -e HF_HUB_OFFLINE=1 \ -e TRANSFORMERS_OFFLINE=1 \ -e HF_DATASETS_OFFLINE=1 \ + -e TORCH_DEVICE_BACKEND_AUTOLOAD=0 \ "${DOCKER_CLIENT_IMAGE}" \ python -m sglang.bench_serving "$@" else diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env index fb65300..87005c0 100644 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env @@ -29,12 +29,12 @@ CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3" USE_DOCKER="${USE_DOCKER:-1}" DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-20260718-sglang}" -# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving; -# reuse the vllm-ascend container itself for the client via `docker exec` (see -# run_adaptive_concurrency_add16.sh), so this is only used if USE_DOCKER_CLIENT=1 -# with an external sglang image. Default off on 910c. -DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" -USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-0}" +# Benchmark client Docker image. The vllm-ascend images built on this host +# (local/vllm-ascend:0.23-a3-*-sglang, see envs/ASCEND_910C_ENV_SETUP.md §8.1) +# carry sglang 0.5.2 bench_serving, so the client runs as a standalone container +# (TORCH_DEVICE_BACKEND_AUTOLOAD=0 injected by the runner, no NPU needed). +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-local/vllm-ascend:0.23-a3-dsv4-sglang}" +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" # Device selection. ASCEND_VISIBLE_DEVICES selects NPU cards 0..7; the Ascend # Docker Runtime (default runtime on this host) injects the matching dies. diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_bench.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_bench.sh index 81e208b..822ca10 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_bench.sh @@ -115,6 +115,7 @@ run_bench_serving() { -e HF_HUB_OFFLINE=1 \ -e TRANSFORMERS_OFFLINE=1 \ -e HF_DATASETS_OFFLINE=1 \ + -e TORCH_DEVICE_BACKEND_AUTOLOAD=0 \ "${DOCKER_CLIENT_IMAGE}" \ python -m sglang.bench_serving "$@" else diff --git a/src/sskj/bench/runner.py b/src/sskj/bench/runner.py index 15caa8d..3c2732a 100644 --- a/src/sskj/bench/runner.py +++ b/src/sskj/bench/runner.py @@ -113,7 +113,16 @@ def run_scenario(scenario: dict[str, Any], options: BenchClientOptions) -> int: cmd += ["-v", f"{options.dataset_path}:{options.dataset_path}:ro"] if options.output_file: cmd += ["-v", f"{options.output_file.parent}:{options.output_file.parent}"] - cmd += ["-e", "HF_HUB_OFFLINE=1", "-e", "TRANSFORMERS_OFFLINE=1", "-e", "HF_DATASETS_OFFLINE=1"] + cmd += [ + "-e", + "HF_HUB_OFFLINE=1", + "-e", + "TRANSFORMERS_OFFLINE=1", + "-e", + "HF_DATASETS_OFFLINE=1", + "-e", + "TORCH_DEVICE_BACKEND_AUTOLOAD=0", + ] cmd += [ options.client_image, "python",