diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md index 7554609..27f66bd 100644 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/README.md @@ -1,26 +1,11 @@ -# GLM-5.2 W4A8 P800 single-node benchmark +# GLM-5.2 W4A8 P800 single-node smoke benchmark -- Validated hosts: `sskj-P800-2` (`gpu049`) and `sskj-P800-4` (`gpu051`) +- Host: `sskj-P800-4` (`gpu051`) - Image: `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746` -- Default model: `/data2/glm52_deploy/models/GLM-5.2-w4a8` +- Model: `/data2/GLM-5.2-w4a8` - Parallelism: TP=8, EP=8, DP=1 - Quantization: `w4a8_int4`; KV cache: `int8` -- Default dataset: `/data2/ShareGPT_V3_unfiltered_cleaned_split.json` +- Dataset: `/data1/wxy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json` - Benchmark: ShareGPT, 64 prompts, concurrency 8, output length 128, request rate unlimited Run `bash start_glm52_w4a8.sh`, wait for `http://127.0.0.1:30000/health`, then run `bash run_bench.sh`. - -The launcher accepts environment overrides so that tuning runs change one -variable at a time. Supported knobs include `CUDA_GRAPH_MAX_BS`, -`CHUNKED_PREFILL_SIZE`, `MAX_PREFILL_TOKENS`, `MEM_FRACTION_STATIC`, -`MAX_RUNNING_REQUESTS`, `KV_CACHE_DTYPE`, and `ENABLE_SPECULATIVE`. - -Run the fixed four-point tuning smoke suite with: - -```bash -RUN_ID= bash run_tuning_smoke.sh -``` - -For EAGLE/MTP, start the service with `ENABLE_SPECULATIVE=1`. The default -speculative settings follow the vendor deployment document: 3 steps, top-k 1, -and 4 draft tokens. diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh index ef944bb..e762d81 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_bench.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh index 9fa2dab..33832cb 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_concurrency_sweep.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-sharegpt-sweep}" RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh index 03cf602..b93ac02 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_full_matrix.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" REFERENCE_DIR="/data1/wxy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-full-matrix}" RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh deleted file mode 100755 index 70d3844..0000000 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/run_tuning_smoke.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -CONTAINER="${CONTAINER:-glm52-w4a8-tp8}" -PORT="${PORT:-30000}" -RUN_DIR="${RUN_DIR:-/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node}" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')-tuning-smoke}" -RESULT_DIR="${RUN_DIR}/results/${RUN_ID}" -RAW_DIR="${RESULT_DIR}/raw_outputs" -LOG_DIR="${RESULT_DIR}/logs" -PYTHON="/root/miniconda/envs/python310_torch25_cuda/bin/python" -SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-2400}" - -SCENARIOS=( - "Y 1024 128 8 40" - "Y 1024 128 32 160" - "Y 8192 128 8 40" - "Y 16384 1024 1 5" -) - -mkdir -p "${RAW_DIR}" "${LOG_DIR}" -curl --fail --silent --show-error --max-time 10 "http://127.0.0.1:${PORT}/health" >/dev/null -docker inspect "${CONTAINER}" >"${RESULT_DIR}/container_inspect.json" -docker logs "${CONTAINER}" >"${RESULT_DIR}/server_before.log" 2>&1 -xpu-smi >"${RESULT_DIR}/xpu_before.txt" - -printf 'mark\tinput_len\toutput_len\tconcurrency\tnum_prompts\n' >"${RESULT_DIR}/scenarios.tsv" -for scenario_row in "${SCENARIOS[@]}"; do - read -r mark isl osl concurrency num_prompts <<<"${scenario_row}" - printf '%s\t%s\t%s\t%s\t%s\n' "${mark}" "${isl}" "${osl}" "${concurrency}" "${num_prompts}" >>"${RESULT_DIR}/scenarios.tsv" -done - -for scenario_row in "${SCENARIOS[@]}"; do - read -r mark isl osl concurrency num_prompts <<<"${scenario_row}" - scenario="c${concurrency}_i${isl}_o${osl}" - started="$(date +%s)" - bench_rc=0 - echo "start ${scenario} prompts=${num_prompts}" - timeout "${SCENARIO_TIMEOUT_S}" docker exec "${CONTAINER}" env \ - HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ - "${PYTHON}" -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 \ - --port "${PORT}" \ - --dataset-name random \ - --dataset-path /workspace/ShareGPT_V3_unfiltered_cleaned_split.json \ - --model /models \ - --served-model-name GLM-5.2-w4a8 \ - --tokenizer /models \ - --random-input-len "${isl}" \ - --random-output-len "${osl}" \ - --random-range-ratio 1.0 \ - --num-prompts "${num_prompts}" \ - --max-concurrency "${concurrency}" \ - --request-rate 10000 \ - --seed 1 \ - --flush-cache \ - --output-file "/workspace/run/results/${RUN_ID}/raw_outputs/${scenario}.jsonl" \ - --output-details \ - >"${LOG_DIR}/${scenario}.log" 2>&1 || bench_rc=$? - duration=$(( $(date +%s) - started )) - if [[ "${bench_rc}" -eq 0 ]]; then - printf 'completed\trc=0\tduration_s=%s\n' "${duration}" >"${LOG_DIR}/${scenario}.status" - else - printf 'failed\trc=%s\tduration_s=%s\n' "${bench_rc}" "${duration}" >"${LOG_DIR}/${scenario}.status" - fi - echo "finish ${scenario} rc=${bench_rc} duration_s=${duration}" - if [[ "${bench_rc}" -ne 0 ]]; then - docker logs --tail 300 "${CONTAINER}" >"${LOG_DIR}/${scenario}.server_tail.log" 2>&1 || true - exit "${bench_rc}" - fi -done - -python3 "${RUN_DIR}/summarize_matrix.py" \ - --scenarios "${RESULT_DIR}/scenarios.tsv" \ - --raw-dir "${RAW_DIR}" \ - --log-dir "${LOG_DIR}" \ - --output "${RESULT_DIR}/summary.tsv" - -docker logs "${CONTAINER}" >"${RESULT_DIR}/server_after.log" 2>&1 -xpu-smi >"${RESULT_DIR}/xpu_after.txt" -echo "tuning_smoke_complete result_dir=${RESULT_DIR}" diff --git a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh index fa66309..09eac6b 100755 --- a/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh +++ b/experiments/p800/glm52_w4a8_sglang_tp8_single_node/start_glm52_w4a8.sh @@ -1,32 +1,12 @@ #!/usr/bin/env bash set -Eeuo pipefail -IMAGE="${IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746}" -MODEL_DIR="${MODEL_DIR:-/data2/glm52_deploy/models/GLM-5.2-w4a8}" -DATASET="${DATASET:-/data2/ShareGPT_V3_unfiltered_cleaned_split.json}" -RUN_DIR="${RUN_DIR:-/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_single_node}" -CONTAINER="${CONTAINER:-glm52-w4a8-tp8}" +IMAGE="iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260715_5746" +MODEL_DIR="/data2/GLM-5.2-w4a8" +DATASET="/data1/wxy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json" +RUN_DIR="/data1/wxy/sskj/experiments/p800/glm52_w4a8_sglang_tp8_smoke" +CONTAINER="glm52-w4a8-tp8" PORT="${PORT:-30000}" -CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:-32768}" -MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-32768}" -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.82}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" -CUDA_GRAPH_MAX_BS="${CUDA_GRAPH_MAX_BS:-8}" -KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-int8}" -ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-0}" -SPECULATIVE_NUM_STEPS="${SPECULATIVE_NUM_STEPS:-3}" -SPECULATIVE_EAGLE_TOPK="${SPECULATIVE_EAGLE_TOPK:-1}" -SPECULATIVE_NUM_DRAFT_TOKENS="${SPECULATIVE_NUM_DRAFT_TOKENS:-4}" - -SPECULATIVE_ARGS=() -if [[ "${ENABLE_SPECULATIVE}" == "1" ]]; then - SPECULATIVE_ARGS+=( - --speculative-algorithm EAGLE - --speculative-num-steps "${SPECULATIVE_NUM_STEPS}" - --speculative-eagle-topk "${SPECULATIVE_EAGLE_TOPK}" - --speculative-num-draft-tokens "${SPECULATIVE_NUM_DRAFT_TOKENS}" - ) -fi test -f "${MODEL_DIR}/config.json" test -f "${DATASET}" @@ -64,23 +44,22 @@ docker run -d \ --trust-remote-code \ --disable-custom-all-reduce \ --disable-shared-experts-fusion \ - --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} \ + --chunked-prefill-size 32768 \ --page-size 64 \ - --mem-fraction-static ${MEM_FRACTION_STATIC} \ - --max-prefill-tokens ${MAX_PREFILL_TOKENS} \ - --max-running-requests ${MAX_RUNNING_REQUESTS} \ + --mem-fraction-static 0.82 \ + --max-prefill-tokens 32768 \ + --max-running-requests 64 \ --tp-size 8 \ --ep-size 8 \ --quantization w4a8_int4 \ - --kv-cache-dtype ${KV_CACHE_DTYPE} \ - --cuda-graph-max-bs ${CUDA_GRAPH_MAX_BS} \ + --kv-cache-dtype int8 \ + --cuda-graph-max-bs 8 \ --disable-cuda-graph-padding \ --watchdog-timeout 3000000 \ --tool-call-parser glm47 \ --reasoning-parser glm45 \ --json-model-override-args '{\"index_topk_freq\": 4}' \ --allow-auto-truncate \ - --enable-metrics \ - ${SPECULATIVE_ARGS[*]}" + --enable-metrics" -echo "container=${CONTAINER} port=${PORT} speculative=${ENABLE_SPECULATIVE} graph_bs=${CUDA_GRAPH_MAX_BS} chunk=${CHUNKED_PREFILL_SIZE} max_prefill=${MAX_PREFILL_TOKENS} mem_fraction=${MEM_FRACTION_STATIC}" +echo "container=${CONTAINER} port=${PORT}"