From 0d929948a5cf016eb207d081e5f64cc211d19e0c Mon Sep 17 00:00:00 2001 From: yy-fighting <2351884576@qq.com> Date: Wed, 9 Sep 2026 15:54:52 +0800 Subject: [PATCH] 60.4 redeploy: TP8+EAGLE+custom-AR 1stage (E7b recipe) replaces TP2PP4 (09-09) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CURRENT.md 60.4 行更新为 E7b 配方在役（EAGLE 4/1/5/mem0.90/MRR16/hicache3/图桶1-8， CAR 补丁 8 rank 激活，质量门 7/7）。deploy_glm53_604_exp.sh 入库（=607 实验版逐字拷贝， 仅改 header；补丁 md5 与 car_patch/ 归档一致）。 注：本提交同时带入并行会话对 CURRENT.md 60.5（v2 容量脚本交付）与 60.8（TP2PP4-nomtp 容量口径在役）两行的未提交更新，内容为当日实测状态。 --- deploy/CURRENT.md | 2 +- .../scripts/deploy_glm53_604_exp.sh | 71 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_dual_scenario_bench/scripts/deploy_glm53_604_exp.sh diff --git a/deploy/CURRENT.md b/deploy/CURRENT.md index 0065fe7..d87f17b 100644 --- a/deploy/CURRENT.md +++ b/deploy/CURRENT.md @@ -10,7 +10,7 @@ | 60.1 (6000D-1) | `glm53-pp4`（Up，8 卡满载，:30000） | **方案 D 生产**。09-08 PD 压测窗口停机 ~2.5h 后已恢复并核验 | `profiles/pro6000/glm53_nvfp4_pro6000_sglang_tp2pp4.env` | | 60.2 (6000D-2) | `glm53-nvfp4` 实验容器（09-08 晚 TP1PP8 phase，run_phase.sh 实验链进行中） | **他人实验进行中，勿动**（方案 F PD 链已拆除；GPU7 曾有外部裸金属任务）。动卡前仍先核实归属 | `profiles/pro6000/glm53_nvfp4_pro6000_pd_{prefill→decode 侧}.env` | | 60.3 | 无容器，但 8 卡被外部裸金属训练占用（`/data/mas/larm`，09-08 晚实测） | 外部任务，勿动（此前台账漏记） | — | -| 60.4 | `glm53-nvfp4`（Up，:30000，restart=unless-stopped，09-09 部署） | **TP2PP4 GLM-5.3-NVFP4 在役**（60.1 方案 D 生产配方复刻：TP2/PP4/mem0.85/MRR48/chunk16384/radix-off/无投机/SM120 三件套/disable-custom-all-reduce/index_topk_freq=4，镜像 nightly-dev-20260828-daf63171；health 200、生成冒烟通过）。09-09 经授权清退外部 vllm 评测流水线（tmux `mas` 的 run_multiseed.sh 链，带 --resume 可续跑）后部署；启动脚本 `/root/deploy_s2_test_604.sh`（基于 exp602/deploy_s2_test.sh，唯一改动 restart=unless-stopped）。旧 `dsv4_scan` 容器仍在盘（Exited） | `profiles/pro6000/glm53_nvfp4_pro6000_sglang_tp2pp4.env`（同 60.1 D 口径） | +| 60.4 | `glm53-nvfp4`（Up，:30000，restart=unless-stopped） | **TP8+EAGLE+custom-AR 1stage（E7b 配方）在役**（09-09 下午部署：EAGLE 4/1/5/mem0.90/MRR16/chunk8192/ctxlen270336/fp8KV+hicache3/decode 图桶 1-8/双 parser；CAR 补丁三处全注入、8 rank `SSKJ_CAR_PATCH_ACTIVE` 确认，health 200、生成冒烟、质量门 7/7 见 `/root/qg_604_car.log`）。启动 `/root/deploy_glm53_604_exp.sh`（=60.7 实验版逐字拷贝，`RESTART=yes CAR_PATCH=1`），补丁 `/root/patches/`（md5 与仓库 car_patch 归档一致）。当日早间曾短暂部署 TP2PP4 D 配方复刻（deploy_s2_test_604.sh 留盘可切回）后被本方案替换；同日经授权清退外部 vllm 评测流水线（tmux `mas` 的 run_multiseed.sh 链，--resume 可续跑） | `experiments/pro6000/glm53_nvfp4_pro6000d_sglang_dual_scenario_bench/scripts/`（deploy_glm53_604_exp.sh + car_patch/ 补丁快照） | | 60.5 | `glm53-nvfp4`（Up 29h） | **NVFP4 团队生产**（deploy_glm53_605.sh，md5 fcd9109b）。生产机铁律：不实验、不重启、不覆盖脚本。**09-09 已交付容量扩容 v2 脚本（TP2PP4-hicache 优胜配置，池 909,632/c6/单条 909k，与 60.8 现役同款），未执行，择维护窗口跑 `/root/deploy_glm53_605_v2.sh`；回滚=原脚本** | `profiles/pro6000/glm53_nvfp4_pro6000_sglang_tp8eagle.env`（方案 A 口径）；待切 `glm53_nvfp4_pro6000_sglang_tp2pp4_hicache.env` | | 60.6 | 无容器，但 8 卡被外部裸金属实验占用（`/data/hzy/sparse-opd-*`，09-08 晚实测） | 外部任务，勿动（此前台账漏记） | — | | 60.7 | 基本空（4 卡仍有 `/home/user/dirA_exp` 外部小任务，09-08 晚实测） | 09-08 已拆除清空（方案 F 前身单机实验 + 场景一深优资产留盘），不再恢复 | — | diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_dual_scenario_bench/scripts/deploy_glm53_604_exp.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_dual_scenario_bench/scripts/deploy_glm53_604_exp.sh new file mode 100644 index 0000000..f710d05 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_dual_scenario_bench/scripts/deploy_glm53_604_exp.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# deploy_glm53_604_exp.sh — GLM-5.3-NVFP4 TP8 + EAGLE + custom-AR 1stage (E7b) deploy for 174.1.60.4 +# Verbatim logic copy of 60.7:/root/deploy_glm53_607_exp.sh (E7b winner recipe, +# scenario-1 campaign 2026-09-07; base = deploy_glm53_605.sh production standard), +# only this header adapted for the 09-09 60.4 redeploy. Patch files already at +# /root/patches/ (md5-verified copies from 60.7). +# +# Env overrides: +# MEMFRAC (0.90) STEPS (4) TOPK (1) DRAFT (5) CTXLEN (270336) +# CHUNK (8192) MAXPRE (16384) RESTART (no|yes) EXTRA ("") +# CAR_PATCH (1) inject custom-AR 1stage patch (E7b winner, 2026-09-07): +# small ARs <=8MB go one-shot kernel instead of NCCL RING_LL on 8-way PCIe. +# Patch files at /root/patches/ (build: bash /root/prep_car_patch.sh on a +# running stock container). Patch dies with the container; image untouched. +# Set CAR_PATCH=0 to deploy stock. +set -e +MEMFRAC=${MEMFRAC:-0.90} +STEPS=${STEPS:-4} +TOPK=${TOPK:-1} +DRAFT=${DRAFT:-5} +CTXLEN=${CTXLEN:-270336} +CHUNK=${CHUNK:-8192} +MAXPRE=${MAXPRE:-16384} +EXTRA=${EXTRA:-} +if [ "$RESTART" = "yes" ]; then RP="--restart unless-stopped"; else RP="--restart no"; fi + +echo "[deploy] removing old container (if any)" +# rm -f times out on big GPU containers on this daemon; retry until really gone +for i in $(seq 1 45); do + CID=$(docker ps -a --filter name=glm53-nvfp4 -q) + [ -z "$CID" ] && break + docker rm -f glm53-nvfp4 >/dev/null 2>&1 || true + sleep 2 +done + +echo "[deploy] waiting for VRAM drain" +for i in $(seq 1 45); do + used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '{s+=$1} END {print s}') + [ "$used" -lt 2000 ] && break + sleep 2 +done +used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '{s+=$1} END {print s}') +echo "[deploy] VRAM now: ${used} MiB total" + +echo "[deploy] starting: TP8 EAGLE ${STEPS}/${TOPK}/${DRAFT} memfrac=${MEMFRAC} chunk=${CHUNK} extra='${EXTRA}'" +docker run -d --name glm53-nvfp4 --gpus all --shm-size 64g --ipc=host $RP \ + -p 30000:30000 -v /data/hf_models:/data/hf_models \ + lmsysorg/sglang:nightly-dev-20260828-daf63171 \ + python3 -m sglang.launch_server \ + --model-path /data/hf_models/GLM-5.3-NVFP4 --tp 8 \ + --mem-fraction-static $MEMFRAC --max-running-requests 16 \ + --chunked-prefill-size $CHUNK --max-prefill-tokens $MAXPRE \ + --disable-shared-experts-fusion --moe-runner-backend flashinfer_cutlass --disable-flashinfer-autotune \ + --speculative-algorithm EAGLE --speculative-num-steps $STEPS --speculative-eagle-topk $TOPK --speculative-num-draft-tokens $DRAFT \ + --kv-cache-dtype fp8_e4m3 --enable-hierarchical-cache --hicache-ratio 3 \ + --cuda-graph-max-bs-decode 8 --cuda-graph-bs-decode 1 2 3 4 6 8 --cuda-graph-max-bs-prefill 8 \ + --context-length $CTXLEN --reasoning-parser glm45 --tool-call-parser glm47 \ + --host 0.0.0.0 --port 30000 $EXTRA + +echo "[deploy] container started; poll: docker logs -f glm53-nvfp4" + +# --- CAR 1stage patch injection (E7b winner) --- +if [ "${CAR_PATCH:-1}" != "0" ] && [ -f /root/patches/custom_all_reduce.py ]; then + echo "[deploy] CAR_PATCH: injecting custom-AR 1stage patch" + docker stop -t 20 glm53-nvfp4 >/dev/null 2>&1 || true + DPATH=/sgl-workspace/sglang/python/sglang/srt/distributed/device_communicators + docker cp /root/patches/custom_all_reduce.py glm53-nvfp4:$DPATH/custom_all_reduce.py + docker cp /root/patches/custom_all_reduce_utils.py glm53-nvfp4:$DPATH/custom_all_reduce_utils.py + docker start glm53-nvfp4 + echo "[deploy] CAR_PATCH injected, container restarted; poll health as usual" +fi