From ba20973beaed07e6f5ccf1be10e9a78e31d614e0 Mon Sep 17 00:00:00 2001 From: Spike Date: Thu, 20 Aug 2026 07:27:02 +0000 Subject: [PATCH] feat(p800): parameterize GLM5.2 single-node tuning --- .../README.md | 23 +++++- .../run_bench.sh | 2 +- .../run_concurrency_sweep.sh | 2 +- .../run_full_matrix.sh | 2 +- .../run_tuning_smoke.sh | 82 +++++++++++++++++++ .../start_glm52_w4a8.sh | 47 ++++++++--- 6 files changed, 138 insertions(+), 20 deletions(-) create mode 100755 experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md index 27f66bd..7554609 100644 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md @@ -1,11 +1,26 @@ -# GLM-5.2 W4A8 P800 single-node smoke benchmark +# GLM-5.2 W4A8 P800 single-node benchmark -- Host: `sskj-P800-4` (`gpu051`) +- Validated hosts: `sskj-P800-2` (`gpu049`) and `sskj-P800-4` (`gpu051`) - Image: `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746` -- Model: `/data2/GLM-5.2-w4a8` +- Default model: `/data2/glm52_deploy/models/GLM-5.2-w4a8` - Parallelism: TP=8, EP=8, DP=1 - Quantization: `w4a8_int4`; KV cache: `int8` -- Dataset: `/data1/wxy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json` +- Default dataset: `/data2/ShareGPT_V3_unfiltered_cleaned_split.json` - Benchmark: ShareGPT, 64 prompts, concurrency 8, output length 128, request rate unlimited Run `bash start_glm52_w4a8.sh`, wait for `http://127.0.0.1:30000/health`, then run `bash run_bench.sh`. + +The launcher accepts environment overrides so that tuning runs change one +variable at a time. Supported knobs include `CUDA_GRAPH_MAX_BS`, +`CHUNKED_PREFILL_SIZE`, `MAX_PREFILL_TOKENS`, `MEM_FRACTION_STATIC`, +`MAX_RUNNING_REQUESTS`, `KV_CACHE_DTYPE`, and `ENABLE_SPECULATIVE`. + +Run the fixed four-point tuning smoke suite with: + +```bash +RUN_ID= bash run_tuning_smoke.sh +``` + +For EAGLE/MTP, start the service with `ENABLE_SPECULATIVE=1`. The default +speculative settings follow the vendor deployment document: 3 steps, top-k 1, +and 4 draft tokens. diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh index e762d81..ef944bb 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh index 33832cb..9fa2dab 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-sharegpt-sweep}" RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh index b93ac02..03cf602 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node" REFERENCE_DIR="/data1/wxy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-full-matrix}" RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh new file mode 100755 index 0000000..70d3844 --- /dev/null +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +CONTAINER="${CONTAINER:-glm52-w4a8-tp8}" +PORT="${PORT:-30000}" +RUN_DIR="${RUN_DIR:-/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node}" +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-tuning-smoke}" +RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" +RAW_DIR="${RESULT_DIR}/raw_outputs" +LOG_DIR="${RESULT_DIR}/logs" +PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-2400}" + +SCENARIOS=( + "Y 1024 128 8 40" + "Y 1024 128 32 160" + "Y 8192 128 8 40" + "Y 16384 1024 1 5" +) + +mkdir -p "${RAW_DIR}" "${LOG_DIR}" +curl --fail --silent --show-error --max-time 10 "http://127.0.0.1:${PORT}/health" >/dev/null +docker inspect "${CONTAINER}" >"${RESULT_DIR}/container_inspect.json" +docker logs "${CONTAINER}" >"${RESULT_DIR}/server_before.log" 2>&1 +xpu-smi >"${RESULT_DIR}/xpu_before.txt" + +printf 'mark\tinput_len\toutput_len\tconcurrency\tnum_prompts\n' >"${RESULT_DIR}/scenarios.tsv" +for scenario_row in "${SCENARIOS[@]}"; do + read -r mark isl osl concurrency num_prompts <<<"${scenario_row}" + printf '%s\t%s\t%s\t%s\t%s\n' "${mark}" "${isl}" "${osl}" "${concurrency}" "${num_prompts}" >>"${RESULT_DIR}/scenarios.tsv" +done + +for scenario_row in "${SCENARIOS[@]}"; do + read -r mark isl osl concurrency num_prompts <<<"${scenario_row}" + scenario="c${concurrency}_i${isl}_o${osl}" + started="$(date +%s)" + bench_rc=0 + echo "start ${scenario} prompts=${num_prompts}" + timeout "${SCENARIO_TIMEOUT_S}" docker exec "${CONTAINER}" env \ + HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "${PYTHON}" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "${PORT}" \ + --dataset-name random \ + --dataset-path /workspace/ShareGPT_V3_unfiltered_cleaned_split.json \ + --model /models \ + --served-model-name GLM-5.2-w4a8 \ + --tokenizer /models \ + --random-input-len "${isl}" \ + --random-output-len "${osl}" \ + --random-range-ratio 1.0 \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate 10000 \ + --seed 1 \ + --flush-cache \ + --output-file "/workspace/run/results/${RUN_ID}/raw_outputs/${scenario}.jsonl" \ + --output-details \ + >"${LOG_DIR}/${scenario}.log" 2>&1 || bench_rc=$? + duration=$(( $(date +%s) - started )) + if [[ "${bench_rc}" -eq 0 ]]; then + printf 'completed\trc=0\tduration_s=%s\n' "${duration}" >"${LOG_DIR}/${scenario}.status" + else + printf 'failed\trc=%s\tduration_s=%s\n' "${bench_rc}" "${duration}" >"${LOG_DIR}/${scenario}.status" + fi + echo "finish ${scenario} rc=${bench_rc} duration_s=${duration}" + if [[ "${bench_rc}" -ne 0 ]]; then + docker logs --tail 300 "${CONTAINER}" >"${LOG_DIR}/${scenario}.server_tail.log" 2>&1 || true + exit "${bench_rc}" + fi +done + +python3 "${RUN_DIR}/summarize_matrix.py" \ + --scenarios "${RESULT_DIR}/scenarios.tsv" \ + --raw-dir "${RAW_DIR}" \ + --log-dir "${LOG_DIR}" \ + --output "${RESULT_DIR}/summary.tsv" + +docker logs "${CONTAINER}" >"${RESULT_DIR}/server_after.log" 2>&1 +xpu-smi >"${RESULT_DIR}/xpu_after.txt" +echo "tuning_smoke_complete result_dir=${RESULT_DIR}" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh index 09eac6b..fa66309 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh @@ -1,12 +1,32 @@ #!/usr/bin/env bash set -Eeuo pipefail -IMAGE="iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746" -MODEL_DIR="/data2/GLM-5.2-w4a8" -DATASET="/data1/wxy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" -CONTAINER="glm52-w4a8-tp8" +IMAGE="${IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746}" +MODEL_DIR="${MODEL_DIR:-/data2/glm52_deploy/models/GLM-5.2-w4a8}" +DATASET="${DATASET:-/data2/ShareGPT_V3_unfiltered_cleaned_split.json}" +RUN_DIR="${RUN_DIR:-/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node}" +CONTAINER="${CONTAINER:-glm52-w4a8-tp8}" PORT="${PORT:-30000}" +CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:-32768}" +MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-32768}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.82}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" +CUDA_GRAPH_MAX_BS="${CUDA_GRAPH_MAX_BS:-8}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-int8}" +ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-0}" +SPECULATIVE_NUM_STEPS="${SPECULATIVE_NUM_STEPS:-3}" +SPECULATIVE_EAGLE_TOPK="${SPECULATIVE_EAGLE_TOPK:-1}" +SPECULATIVE_NUM_DRAFT_TOKENS="${SPECULATIVE_NUM_DRAFT_TOKENS:-4}" + +SPECULATIVE_ARGS=() +if [[ "${ENABLE_SPECULATIVE}" == "1" ]]; then + SPECULATIVE_ARGS+=( + --speculative-algorithm EAGLE + --speculative-num-steps "${SPECULATIVE_NUM_STEPS}" + --speculative-eagle-topk "${SPECULATIVE_EAGLE_TOPK}" + --speculative-num-draft-tokens "${SPECULATIVE_NUM_DRAFT_TOKENS}" + ) +fi test -f "${MODEL_DIR}/config.json" test -f "${DATASET}" @@ -44,22 +64,23 @@ docker run -d \ --trust-remote-code \ --disable-custom-all-reduce \ --disable-shared-experts-fusion \ - --chunked-prefill-size 32768 \ + --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} \ --page-size 64 \ - --mem-fraction-static 0.82 \ - --max-prefill-tokens 32768 \ - --max-running-requests 64 \ + --mem-fraction-static ${MEM_FRACTION_STATIC} \ + --max-prefill-tokens ${MAX_PREFILL_TOKENS} \ + --max-running-requests ${MAX_RUNNING_REQUESTS} \ --tp-size 8 \ --ep-size 8 \ --quantization w4a8_int4 \ - --kv-cache-dtype int8 \ - --cuda-graph-max-bs 8 \ + --kv-cache-dtype ${KV_CACHE_DTYPE} \ + --cuda-graph-max-bs ${CUDA_GRAPH_MAX_BS} \ --disable-cuda-graph-padding \ --watchdog-timeout 3000000 \ --tool-call-parser glm47 \ --reasoning-parser glm45 \ --json-model-override-args '{\"index_topk_freq\": 4}' \ --allow-auto-truncate \ - --enable-metrics" + --enable-metrics \ + ${SPECULATIVE_ARGS[*]}" -echo "container=${CONTAINER} port=${PORT}" +echo "container=${CONTAINER} port=${PORT} speculative=${ENABLE_SPECULATIVE} graph_bs=${CUDA_GRAPH_MAX_BS} chunk=${CHUNKED_PREFILL_SIZE} max_prefill=${MAX_PREFILL_TOKENS} mem_fraction=${MEM_FRACTION_STATIC}"