feat(p800): parameterize GLM5.2 single-node tuning

2026-08-20 07:27:02 +00:00 · 2026-08-20 07:27:02 +00:00 · ba20973bea
commit ba20973bea
parent eeec56c2cf
6 changed files with 138 additions and 20 deletions
--- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md
+++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md
@ -1,11 +1,26 @@
-# GLM-5.2 W4A8 P800 single-node smoke benchmark
+# GLM-5.2 W4A8 P800 single-node benchmark

- Host: `sskj-P800-4` (`gpu051`)
+- Validated hosts: `sskj-P800-2` (`gpu049`) and `sskj-P800-4` (`gpu051`)
 - Image: `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746`
- Model: `/data2/GLM-5.2-w4a8`
+- Default model: `/data2/glm52_deploy/models/GLM-5.2-w4a8`
 - Parallelism: TP=8, EP=8, DP=1
 - Quantization: `w4a8_int4`; KV cache: `int8`
- Dataset: `/data1/wxy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json`
+- Default dataset: `/data2/ShareGPT_V3_unfiltered_cleaned_split.json`
 - Benchmark: ShareGPT, 64 prompts, concurrency 8, output length 128, request rate unlimited

 Run `bash start_glm52_w4a8.sh`, wait for `http://127.0.0.1:30000/health`, then run `bash run_bench.sh`.
+
+The launcher accepts environment overrides so that tuning runs change one
+variable at a time. Supported knobs include `CUDA_GRAPH_MAX_BS`,
+`CHUNKED_PREFILL_SIZE`, `MAX_PREFILL_TOKENS`, `MEM_FRACTION_STATIC`,
+`MAX_RUNNING_REQUESTS`, `KV_CACHE_DTYPE`, and `ENABLE_SPECULATIVE`.
+
+Run the fixed four-point tuning smoke suite with:
+
+```bash
+RUN_ID=<name> bash run_tuning_smoke.sh
+```
+
+For EAGLE/MTP, start the service with `ENABLE_SPECULATIVE=1`. The default
+speculative settings follow the vendor deployment document: 3 steps, top-k 1,
+and 4 draft tokens.
--- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh
+++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh
@ -3,7 +3,7 @@ set -Eeuo pipefail

 CONTAINER="glm52-w4a8-tp8"
 PORT="${PORT:-30000}"
-RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke"
+RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node"
 RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
 RESULT_DIR="${RUN_DIR}/results/${RUN_ID}"
 PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python"
--- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh
+++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh
@ -3,7 +3,7 @@ set -Eeuo pipefail

 CONTAINER="glm52-w4a8-tp8"
 PORT="${PORT:-30000}"
-RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke"
+RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node"
 RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-sharegpt-sweep}"
 RESULT_DIR="${RUN_DIR}/results/${RUN_ID}"
 PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python"
--- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh
+++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh
@ -3,7 +3,7 @@ set -Eeuo pipefail

 CONTAINER="glm52-w4a8-tp8"
 PORT="${PORT:-30000}"
-RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke"
+RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node"
 REFERENCE_DIR="/data1/wxy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix"
 RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-full-matrix}"
 RESULT_DIR="${RUN_DIR}/results/${RUN_ID}"
--- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh
+++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh
@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+CONTAINER="${CONTAINER:-glm52-w4a8-tp8}"
+PORT="${PORT:-30000}"
+RUN_DIR="${RUN_DIR:-/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node}"
+RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-tuning-smoke}"
+RESULT_DIR="${RUN_DIR}/results/${RUN_ID}"
+RAW_DIR="${RESULT_DIR}/raw_outputs"
+LOG_DIR="${RESULT_DIR}/logs"
+PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python"
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-2400}"
+
+SCENARIOS=(
+  "Y 1024 128 8 40"
+  "Y 1024 128 32 160"
+  "Y 8192 128 8 40"
+  "Y 16384 1024 1 5"
+)
+
+mkdir -p "${RAW_DIR}" "${LOG_DIR}"
+curl --fail --silent --show-error --max-time 10 "http://127.0.0.1:${PORT}/health" >/dev/null
+docker inspect "${CONTAINER}" >"${RESULT_DIR}/container_inspect.json"
+docker logs "${CONTAINER}" >"${RESULT_DIR}/server_before.log" 2>&1
+xpu-smi >"${RESULT_DIR}/xpu_before.txt"
+
+printf 'mark\tinput_len\toutput_len\tconcurrency\tnum_prompts\n' >"${RESULT_DIR}/scenarios.tsv"
+for scenario_row in "${SCENARIOS[@]}"; do
+  read -r mark isl osl concurrency num_prompts <<<"${scenario_row}"
+  printf '%s\t%s\t%s\t%s\t%s\n' "${mark}" "${isl}" "${osl}" "${concurrency}" "${num_prompts}" >>"${RESULT_DIR}/scenarios.tsv"
+done
+
+for scenario_row in "${SCENARIOS[@]}"; do
+  read -r mark isl osl concurrency num_prompts <<<"${scenario_row}"
+  scenario="c${concurrency}_i${isl}_o${osl}"
+  started="$(date +%s)"
+  bench_rc=0
+  echo "start ${scenario} prompts=${num_prompts}"
+  timeout "${SCENARIO_TIMEOUT_S}" docker exec "${CONTAINER}" env \
+    HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \
+    "${PYTHON}" -m sglang.bench_serving \
+    --backend sglang \
+    --host 127.0.0.1 \
+    --port "${PORT}" \
+    --dataset-name random \
+    --dataset-path /workspace/ShareGPT_V3_unfiltered_cleaned_split.json \
+    --model /models \
+    --served-model-name GLM-5.2-w4a8 \
+    --tokenizer /models \
+    --random-input-len "${isl}" \
+    --random-output-len "${osl}" \
+    --random-range-ratio 1.0 \
+    --num-prompts "${num_prompts}" \
+    --max-concurrency "${concurrency}" \
+    --request-rate 10000 \
+    --seed 1 \
+    --flush-cache \
+    --output-file "/workspace/run/results/${RUN_ID}/raw_outputs/${scenario}.jsonl" \
+    --output-details \
+    >"${LOG_DIR}/${scenario}.log" 2>&1 || bench_rc=$?
+  duration=$(( $(date +%s) - started ))
+  if [[ "${bench_rc}" -eq 0 ]]; then
+    printf 'completed\trc=0\tduration_s=%s\n' "${duration}" >"${LOG_DIR}/${scenario}.status"
+  else
+    printf 'failed\trc=%s\tduration_s=%s\n' "${bench_rc}" "${duration}" >"${LOG_DIR}/${scenario}.status"
+  fi
+  echo "finish ${scenario} rc=${bench_rc} duration_s=${duration}"
+  if [[ "${bench_rc}" -ne 0 ]]; then
+    docker logs --tail 300 "${CONTAINER}" >"${LOG_DIR}/${scenario}.server_tail.log" 2>&1 || true
+    exit "${bench_rc}"
+  fi
+done
+
+python3 "${RUN_DIR}/summarize_matrix.py" \
+  --scenarios "${RESULT_DIR}/scenarios.tsv" \
+  --raw-dir "${RAW_DIR}" \
+  --log-dir "${LOG_DIR}" \
+  --output "${RESULT_DIR}/summary.tsv"
+
+docker logs "${CONTAINER}" >"${RESULT_DIR}/server_after.log" 2>&1
+xpu-smi >"${RESULT_DIR}/xpu_after.txt"
+echo "tuning_smoke_complete result_dir=${RESULT_DIR}"
--- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh
+++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh
@ -1,12 +1,32 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail

-IMAGE="iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746"
-MODEL_DIR="/data2/GLM-5.2-w4a8"
-DATASET="/data1/wxy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json"
-RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke"
-CONTAINER="glm52-w4a8-tp8"
+IMAGE="${IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746}"
+MODEL_DIR="${MODEL_DIR:-/data2/glm52_deploy/models/GLM-5.2-w4a8}"
+DATASET="${DATASET:-/data2/ShareGPT_V3_unfiltered_cleaned_split.json}"
+RUN_DIR="${RUN_DIR:-/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node}"
+CONTAINER="${CONTAINER:-glm52-w4a8-tp8}"
 PORT="${PORT:-30000}"
+CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:-32768}"
+MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-32768}"
+MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.82}"
+MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
+CUDA_GRAPH_MAX_BS="${CUDA_GRAPH_MAX_BS:-8}"
+KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-int8}"
+ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-0}"
+SPECULATIVE_NUM_STEPS="${SPECULATIVE_NUM_STEPS:-3}"
+SPECULATIVE_EAGLE_TOPK="${SPECULATIVE_EAGLE_TOPK:-1}"
+SPECULATIVE_NUM_DRAFT_TOKENS="${SPECULATIVE_NUM_DRAFT_TOKENS:-4}"
+
+SPECULATIVE_ARGS=()
+if [[ "${ENABLE_SPECULATIVE}" == "1" ]]; then
+  SPECULATIVE_ARGS+=(
+    --speculative-algorithm EAGLE
+    --speculative-num-steps "${SPECULATIVE_NUM_STEPS}"
+    --speculative-eagle-topk "${SPECULATIVE_EAGLE_TOPK}"
+    --speculative-num-draft-tokens "${SPECULATIVE_NUM_DRAFT_TOKENS}"
+  )
+fi

 test -f "${MODEL_DIR}/config.json"
 test -f "${DATASET}"
@ -44,22 +64,23 @@ docker run -d \
    --trust-remote-code \
    --disable-custom-all-reduce \
    --disable-shared-experts-fusion \
-    --chunked-prefill-size 32768 \
+    --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} \
    --page-size 64 \
-    --mem-fraction-static 0.82 \
-    --max-prefill-tokens 32768 \
-    --max-running-requests 64 \
+    --mem-fraction-static ${MEM_FRACTION_STATIC} \
+    --max-prefill-tokens ${MAX_PREFILL_TOKENS} \
+    --max-running-requests ${MAX_RUNNING_REQUESTS} \
    --tp-size 8 \
    --ep-size 8 \
    --quantization w4a8_int4 \
-    --kv-cache-dtype int8 \
-    --cuda-graph-max-bs 8 \
+    --kv-cache-dtype ${KV_CACHE_DTYPE} \
+    --cuda-graph-max-bs ${CUDA_GRAPH_MAX_BS} \
    --disable-cuda-graph-padding \
    --watchdog-timeout 3000000 \
    --tool-call-parser glm47 \
    --reasoning-parser glm45 \
    --json-model-override-args '{\"index_topk_freq\": 4}' \
    --allow-auto-truncate \
-    --enable-metrics"
+    --enable-metrics \
+    ${SPECULATIVE_ARGS[*]}"

-echo "container=${CONTAINER} port=${PORT}"
+echo "container=${CONTAINER} port=${PORT} speculative=${ENABLE_SPECULATIVE} graph_bs=${CUDA_GRAPH_MAX_BS} chunk=${CHUNKED_PREFILL_SIZE} max_prefill=${MAX_PREFILL_TOKENS} mem_fraction=${MEM_FRACTION_STATIC}"