From 9e5640138463ff8d75ced0c192766f701d80df61 Mon Sep 17 00:00:00 2001 From: yy-fighting <2351884576@qq.com> Date: Mon, 7 Sep 2026 02:43:31 +0800 Subject: [PATCH] PP+MTP deepdive r3: race bisect (mask tooling, r34 candidate), decode-round quantification (gloo rendezvous stalls + AR spin dominate; 3-source hypothesis refuted), bench-profile crash forensics --- .../README.md | 40 + ...M53_TP4PP2_Profile与优化方向_2026-09-06.md | 267 ++ .../analysis/ppmtp_wiki_report_0906.xml | 62 + .../results/ops/prof_mtp_analysis.txt | 216 ++ .../scripts/deploy_ppmtp_mask.sh | 70 + .../scripts/deploy_ppmtp_r34.sh | 70 + .../scripts/deploy_ppmtp_r34_nocache.sh | 70 + .../scripts/eagle_worker_v2_mask.py | 1962 +++++++++++++++ .../scripts/prof_ann_census.py | 31 + .../scripts/prof_mtp_analyze.py | 104 + .../scripts/prof_mtp_decode.sh | 47 + .../scripts/prof_mtp_decode2.sh | 43 + .../scripts/prof_mtp_decode3.sh | 48 + .../scripts/race_run.sh | 46 + .../scripts/scheduler_pp_mixin_r34.py | 2156 +++++++++++++++++ 15 files changed, 5232 insertions(+) create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/README.md create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/GLM53_TP4PP2_Profile与优化方向_2026-09-06.md create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/ppmtp_wiki_report_0906.xml create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/ops/prof_mtp_analysis.txt create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_mask.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34_nocache.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/eagle_worker_v2_mask.py create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_ann_census.py create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_analyze.py create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode2.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode3.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/race_run.sh create mode 100644 experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/scheduler_pp_mixin_r34.py diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/README.md b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/README.md new file mode 100644 index 0000000..6b5e1e2 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/README.md @@ -0,0 +1,40 @@ +# GLM-5.3-NVFP4 TP4PP2+MTP 深入攻坚（2026-09-07 第三轮） + +机器：174.1.60.5（8×RTX 6000D）。基础补丁 = 09-06 的 r33（sglang_patch2 七文件挂载 + nightly-dev-cu13-20260901-07c8f729），本轮在其上做竞态二分、修复尝试与 decode 轮量化。主报告（结论与工程清单）：`results/analysis/GLM53_TP4PP2_Profile与优化方向_2026-09-06.md` §9。 + +## scripts/ + +| 文件 | 作用 | +|---|---| +| eagle_worker_v2_mask.py | r33 的位掩码版：7 处阶段边界同步改为 `/tmp/pp_sync_mask` 热开关（bit0-6 = site1-7：draftstep/prefill-ext/ext-pre/extlogits/draft-tree/accept/replay-in），缺省 127 ≡ r33 | +| scheduler_pp_mixin_r34.py | 修复候选：`_pp_commit_comm_work` 延迟一轮释放 P2PWork（实验结果：反而加速竞态触发，弃用） | +| deploy_ppmtp_mask.sh | 部署掩码构建（挂 mask 版 eagle_worker_v2） | +| deploy_ppmtp_r34.sh | 部署 r34（挂 r34 mixin + 健康后写掩码文件 SYNCMASK） | +| deploy_ppmtp_r34_nocache.sh | r34 + PYTORCH_NO_CUDA_MEMORY_CACHING=1（死于启动期 graph capture，实验不可行） | +| race_run.sh | 竞态单点运行器：热写掩码 → conc_test big → 监控容器/健康死亡并收割日志 | +| prof_mtp_decode{,2,3}.sh | torch profiler 采集编排（12 步 spec round，无 stack） | +| prof_mtp_analyze.py | trace 分析：轮时拆解、cudaStreamSynchronize/cudaLaunchKernel 宿主成本、NCCL 分桶、gap 归因、长宿主 op | +| prof_ann_census.py | user_annotation 相位普查（TARGET_VERIFY/draft/DECODE/边界收发时长 + gloo 操作画像） | + +## results/ops/（实验原始记录） + +- race_r0.log：mask=127 对照，conc_test 8×16384 PASS 64.4s +- race_r1.log / race_r1b_ladder.log / race_r1c_big.log：mask=0 conc_test 3/3 PASS（30.7/27.5/26.6s）——**假信心**，bench 才触发竞态 +- crash_mask0_full.log：r33+mask0 完整 bench cc16 崩溃（prefill 爬坡段，PG2 watchdog IMA，表面 dsa init/allocator insert_events 同 09-06） +- crash_r34_mask0.log：r34+mask0 conc_test 23.6s 崩（commit wait 也是隐性遮罩的直接证据） +- deploy_r34.log / deploy_r34nc.log：部署记录（后者 = NO_CACHING 死于 cudaErrorStreamCaptureInvalidated） +- prof_mtp_decode.log / prof_mtp_decode3.log：采集编排日志 +- prof_mtp_analysis.txt：8 rank trace 全量分析输出 + +## 核心事实（详见主报告 §9） + +1. 三源证伪：7 同步 4.9ms/轮、eager launch 3.3ms/轮、边界 SendRecv 0.17ms/轮——合计 <10ms，解释不了 240ms 回归。 +2. 真凶：每轮 3 次串行 GLOO 元数据 rendezvous（parallel_state.send_object = 2 gloo isend/字典），相位漂移下发送端 CPU 阻塞 300-540ms → TP 到达偏斜 → AR 自旋 132ms/轮 + GPU 空洞 ~130ms/轮（293.5ms/轮的 72% 是 TARGET_VERIFY 的等待态）。 +3. 竞态为真且与停顿去除耦合：mask=0 在 bench 剖面崩；r34 去 commit wait 更快崩。根因定位只能靠 compute-sanitizer（NO_CACHING/LAUNCH_BLOCKING 均引入同步污染）。 +4. 16k 输入下 KV 池把 decode 并发钉在 ≤8：cc16 压测的 decode 实际是 bs 7-8 混合态，"稳态 bs16"不存在。 + +## 远端留存（60.5） + +- /root/sglang_patch2/（r33 七文件）、/root/eagle_worker_v2_mask.py、/root/scheduler_pp_mixin_r34.py、deploy_ppmtp_{mask,r34,r34_nocache}.sh、race_run.sh、prof_mtp_*.sh/py +- /data/hf_models/ppmtp_profiles/M_decode/（8 rank × 12 步 trace，~48MB） +- 崩溃与竞态日志：/root/crash_*.log、/root/race_*.log diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/GLM53_TP4PP2_Profile与优化方向_2026-09-06.md b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/GLM53_TP4PP2_Profile与优化方向_2026-09-06.md new file mode 100644 index 0000000..33faab2 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/GLM53_TP4PP2_Profile与优化方向_2026-09-06.md @@ -0,0 +1,267 @@ +# GLM-5.3-NVFP4 TP4PP2 Profile 与优化方向 + +日期：2026-09-06（profile）/ 2026-09-07（第二轮优化实验）　|　机器：174.1.60.5（8×RTX 6000D 85.6GB，SM120，PCIe Gen5，无 NVLink）　|　模型：GLM-5.3-NVFP4（78 层 + 1 nextn，hidden 6144，256 专家，first_k_dense 3，kv_lora_rank 512，DSA index_topk 2048） + +## 0. 一句话结论 + +TP4PP2 在 prefill 主导的负载下**没有教科书式 PP 气泡**（chunked prefill 天然形成微批流水，两 stage 同忙 96%）；真正的墙是 **TP4 AllReduce 占 prefill 关键路径 30%**。09-07 第二轮实验把配置层面的快赢**全部排除**：NCCL 协议切换（env 到不了 rank + AR 已近 PCIe 带宽极限）、chunk 16384（MoE 工作区 OOM + 带宽受限下无次数红利）、现成通信开关（量化通信 NPU 专用、AR 融合 SM90/100 门控、symm-mem/NVLS 是 NVLink 向）——**现有配置（chunk 8192 / mem-frac 0.88 / 默认 NCCL）已是本栈的配置最优**；128k 场景的意外头号热点 DSA indexer（28%，1.6× 稀疏注意力本体）经核算属**算法固有成本**（无缓存复用机会、无可切换内核、模型已做跨层 topk 共享）。进一步提升全部是开发投入：优先级 = **PP+MTP 上游化（decode，TPOT -60%+ 潜力）> AR 分块重叠/量化 AR 内核开发（prefill，-15~25% 潜力）**。 + +## 1. 环境与部署忠实性 + +部署命令（与 09-05 对比报告获胜配置逐字一致）： + +``` +bash /root/deploy_par.sh "--tp 4 --pp-size 2 --disable-overlap-schedule --max-prefill-tokens 16384" nomtp +``` + +- 镜像 `lmsysorg/sglang:nightly-dev-20260828-daf63171`，容器 glm53-nvfp4:30000，权重 /data/hf_models/GLM-5.3-NVFP4 +- KV fp8_e4m3 自动；SM120 三件套 `--disable-shared-experts-fusion --moe-runner-backend flashinfer_cutlass --disable-flashinfer-autotune`；hicache-ratio 3；mem-frac 0.88；chunked-prefill-size 8192 +- PP 模式 nightly 硬约束：nomtp + `--disable-overlap-schedule` +- 系统库 NCCL 2.28.3（CUDA 13.0 build），容器未设任何 NCCL 环境变量（默认调参） + +**忠实性验证**：干净无 profiler 复测 cc16 16k/512 —— 输出 104.7 tok/s、TPOT 50.9ms，与 09-05 基准（103.3 tok/s / 55.4ms）一致 → 本机部署复现了对比报告的性能，profile 结论可回推到基准数据。 + +| 指标 | 09-05 基准 | 本次干净复测 | +|---|---|---| +| 输入吞吐 | 3306 tok/s | 3350 tok/s | +| 输出吞吐 | 103.3 tok/s | 104.7 tok/s | +| TTFT | 29.2 s | ~31 s | +| TPOT | 55.4 ms | 50.9 ms | + +## 2. 采集协议 + +torch profiler（sglang `POST /start_profile`，CPU+GPU activities，无 stack），8 rank 全采，精读 PP0-TP0 / PP1-TP0（TP1 做一致性对照）。跨 rank 墙钟对齐用 trace 内 baseTimeNanoseconds。采集前 flush cache + 微量预热 + 再 flush；bench 一律换新 seed（run-id 隔离）。 + +| 场景 | 负载 | 采集窗口 | 步数 | +|---|---|---|---| +| A prefill | 16×(16384→1) cc16 | 全部 prefill forward | ~34 前向 | +| B decode | 16×(16384→512) cc16 | TTFT 后新 "Decode batch" 起 | 60 步 | +| C mapping | with_stack 小采集 | — | **两次均失败**（见 §4.5） | +| D 128k/90% | 8×131072（90% 共享前缀）cc8 | 后缀 prefill 窗口 | 16 前向 | + +trace 留存：`60.5:/data/hf_models/pp2tp4_profiles/{A_prefill,B_decode,D_128k}/`；分析输出：`.../analysis/`；脚本：`60.5:/root/prof_tools/`（本地副本 `D:\sskj\prof_tools\`）。 + +## 3. 核心发现 + +### 3.1 Prefill（A：16×16384→1，cc16，墙钟 35.2s） + +**PP 气泡假说被证伪。** 跨 stage 1ms 分箱对齐：两 stage 同时忙碌 **96.0%**，仅 PP0 忙 1.1%，仅 PP1 忙 2.9%，双闲 0.0%。chunked prefill 把 16 个请求切成 ~34 个 8192-token 前向，天然形成微批流水，PP1 的计算完全藏在 PP0 之后。09-05 报告里 TP4PP2 相对 TP8 的收益**不是**以气泡为代价换来的。 + +**墙钟 = 最慢 stage 的关键链路：** + +| PP0 链路（决定墙钟） | 耗时 | 占比 | +|---|---|---| +| GPU 计算 | 22.8 s | 65% | +| TP4 AllReduce（2686 次 × p50 3.69ms） | 10.4 s | 30% | +| 其余（launch 间隙等） | ~2 s | 5% | + +PP1 链路（计算 21.2s + AR 10.0s + AllGather 0.24s）被流水完全隐藏。级间 SendRecv 真传输 p50 3.33ms×5 次/前向；p90 678ms 的空转档是稳态背压信号，被流水线吸收、不进墙钟。 + +**计算热点（两 stage 一致）：** sparse_mla_prefill 8.5s（~25%）；cutlass NVFP4 MoE GEMM ~5.5s（~17%）；deep_gemm fp8 密集 GEMM ~4s（PP0 ~12%）；MoE 路由胶水（expand/finalize/activation/topk）~2s（~6%）；DSA indexer（mqa_logits+topk）~3%；Memcpy DtoH 568ms（n=166）。 + +**结构性观察：每前向仅 8192 token。** max-prefill-tokens 虽为 16384，但 chunked-prefill-size 8192 封顶了每个前向的有效 token 数——AR 次数（79/前向）、流水线填充、launch 等固定开销只按 8192 token 摊薄。 + +### 3.2 Decode（B：16k KV，bs 7-16，步长 21-26ms，cuda graph 重放） + +| PP0 每步构成 | 耗时 | 占比 | +|---|---|---| +| MoE NVFP4 GEMM（72 次小 batch，均值 106µs） | 7.65 ms | 35% | +| 级间 SendRecv（~5 次 × 0.8ms） | 4 ms | 18% | +| dense bf16 GEMM | ~2.3 ms | 10% | +| TP4 AllReduce（79 次 × 18µs） | 1.4 ms | 6% | +| sparse_mla_decode | 0.8 ms | 4% | + +PP1 每步：MoE GEMM ~9.2ms（42%）、AR 1.9ms、**AllGather 1.4ms（273 次 launch 走 cuLaunchKernelExC，不在 cuda graph 内）**、SendRecv ~1ms。步间 CPU 间隙 p50 2.3-2.6ms（overlap-schedule 被禁的代价，~10%）。 + +- 无 MTP：每步 1 token/请求。干净 TPOT 50.9ms 与步长 21-26ms 的差值 = 交错进入 + 低并发期 + 尾部（最后一个请求独占解码，~28 tok/s）+ CPU 间隙。 +- 采集期内仅 7-8 个请求同时处于 decode（TTFT 错峰进入），说明 cc16 下 decode 相位从未满batch。 +- 带 profiler 时 TPOT 虚高到 97.8ms（2×）——**profiler 采集期的 bench 指标一律作废**，干净指标必须单独跑。 + +### 3.3 128k / 90% 前缀命中（D：cc8，后缀 prefill 窗口 19.9s） + +日志核实命中率 0.8999。后缀有效 prefill 速率 **~5.2k tok/s**（同机 TP8 仅 ~3.2k——TP4PP2 的 KV 容量 2.37× 使全部 KV GPU 常驻，227k < 589k 容量，无需 TP8 必需的 hicache 往返）。 + +| 热点 | PP0 | PP1 | +|---|---|---| +| **DSA indexer `deep_gemm::sm120_fp8_mqa_logits`** | **28%** | **23%** | +| sparse_mla（稀疏注意力本体） | 17% | 17% | +| TP4 AllReduce | 20% | 20% | +| hicache/radix 恢复间隙（p50 77.5ms/前向） | ~5% | ~5% | + +**意外发现：indexer 的 MQA logits GEMM 是它所服务的稀疏注意力的 1.6 倍。** DSA 选索引时对 ~118k 已缓存前缀键逐层算 q·k logits——前缀命中场景下这部分键对每个请求都相同，但当前实现对整个前缀重算。这是 128k 场景 TTFT 的第一大头，也是最值得攻的算法级优化点。 + +### 3.4 NCCL 取证（A） + +- AllReduce 内核名 `ncclDevKernel_AllReduce_Sum_bf16_RING_LL`：~96MB 消息（8192×6144×2B）每次 3.69ms（极均匀，p10-p99 都在 3.68-4.18ms）。 +- 每前向 79 次 AR ≈ 292ms ≈ **29% 的 1.03s 前向**；整个 A 窗口 2686 次 ≈ 10.4s。 +- **带宽核算：96MB / 3.7ms ⇒ algbw 26GB/s，环形 busbw ≈ 39GB/s**，约为 PCIe Gen5 x16 实际大包峰值的 ~75%。真跑半带宽效率的 LL 线格式需要 ~78GB/s 原始速率——超过 PCIe Gen5 理论值，物理不可能。结论：内核名里的 "_LL" 是 NCCL 2.28 混合 plan 的命名，不代表线上是 LL 格式；**AR 实际已以接近最优的有效带宽运行，协议层没有免费午餐**（§5 的强制 Simple 实验证实了这一点）。 +- SendRecv 双峰：p50 3.33ms 真传输 + p90 678ms 空转（流水背压，非墙钟成本）。 +- 环境核查：容器与 launcher 进程均无 NCCL 调参变量（NCCL 2.28.3 全默认调参）。 + +### 3.5 工程发现（profiler 操作性结论，本 nightly） + +1. **profiler flush 阻塞整个流水线**：60 步无栈 decode trace 的 flush 让服务停顿 11s（日志 14:57:28 stop → 14:57:39 done）。采集步数控制在 ≤60 步、bench 指标单独干净跑。 +2. **with_stack 病态（两次复现）**：16k prefill 上开 with_stack → 单 rank RSS 145-173GB、容器 841GB/1008GB、health=000、容器僵尸（docker restart 失败），必须宿主机 kill -9 全部 sglang 进程 + docker rm -f 重部署；2k 小 probe 上开 with_stack → stop 已确认但 flush 5 分钟内不出文件、服务再次拖死。**本版本 with_stack 一律不用**，kernel→源码映射改用经验+符号名。 +3. 采集自动停条件是 forward_ct ≥ start+num_steps+1（+1 守卫步），步数给多了 profiler 会一直武装着等——需要手动 stop_profile（该 POST 会阻塞到 flush 完，不能放在 rsh 链内联）。 + +## 4. 优化方向排序（09-07 第二轮实验后更新） + +| # | 方向 | 状态（09-07 更新） | 预期收益 | 落地难度 | +|---|---|---|---|---| +| 1 | **PP+MTP（decode 最大杠杆）** | 唯一剩余大杠杆。步长 21-26ms 无投机；MTP accept ~2.5；楔死根因已定位（09-05 补丁），需上游化 | TPOT 等效 ~9-10ms/token（-60%+） | 高（开发项目） | +| 2 | **prefill AR 通信-计算重叠 / 量化 AR** | **无现成开关**（§6.3 扫描：quant-comm NPU 专用、AR 融合 SM90/100 门控、symm-mem/NVLS 是 NVLink 向），需上游/内核开发 | AR 逼近零暴露：prefill 墙钟 -15~25% | 中高（分块重叠的 TP linear 改造，或将 AR 融合内核扩展到 SM120+大 token 数） | +| 3 | MoE 胶水融合 + PP1 AllGather 入图 | 上游开发 | decode -2~4% | 中 | +| 4 | decode 低并发/尾部 | overlap-sched 被 PP 禁是 nightly 硬约束 | 长尾 P99 改善 | 中 | + +**已证伪/关闭（记录在案，勿再投入）**： +- **NCCL 协议/算法调参**（§5）：容器 env 到不了 rank 进程，且 AR 已在带宽极限附近（busbw≈39GB/s ≈ PCIe Gen5 实际峰值 ~75%）；即便进程内强制生效上限也只有 prefill ~6%。 +- **chunk 16384**（§6.2）：MoE 工作区随 chunk 增长 OOM（3.08GB > 2.11GB 空闲，需 mem-frac ≤0.85）；且 AR 带宽受限使"次数减半"无红利、indexer 功不变——剩余收益 ≤3%（噪声内）却要付 KV -5% + Max ITL 翻倍。TP8 口径"chunk 8192 勿调大"在 TP4PP2 同样成立。 +- **128k DSA indexer**（§6.4）：28% 属算法固有成本（每请求后缀 q 独有、无跨请求复用；SM120 仅 deepgemm 一个内核后端；模型已做 index_topk_freq=4 跨层共享；kernel ~219 TFLOPS/rank 属 paged-gather 合理量级）。逃逸路径（分层索引）是影响精度的上游研究项。 + +优先级依据：#1 是唯一数量级杠杆且根因/补丁已就绪；#2 是 prefill 唯一剩余的结构性方向（协议红利已排除）；#3-#4 是小额累积项。 + +## 5. 验证实验：NCCL_PROTO=Simple（假设证伪） + +Profile 提出的第一假设是"prefill AR 内核名带 `_RING_LL` → NCCL 在 96MB 大消息上误选 LL 协议（半带宽）→ 强制 Simple 可省 ~11% prefill 墙钟"。为此做了同配置 +1 环境变量的对照实验： + +**设置**：`deploy_par.sh` 逐字相同，仅 docker run 追加 `-e NCCL_PROTO=Simple`（容器 config env 与 launcher 进程 environ 均核实含有该变量）。 + +**结果**： + +| 指标 | 原样（默认调参） | 强制 Simple | 判读 | +|---|---|---|---| +| 输入吞吐（cc16 16k/512） | 3350 tok/s | 3248 tok/s | 无改善，单轮噪声内 | +| 输出吞吐 | 104.7 tok/s | 101.5 tok/s | 同上 | +| Mean TTFT | 31.36 s | 30.71 s | 持平 | +| Mean TPOT | 50.87 ms | 57.54 ms | 单轮波动（chunked prefill 干扰下 TPOT 本身有较大 run 间方差） | +| AR 内核（2 步 prefill trace 取证） | `..._RING_LL` p50 3.69ms ×79/前向 | `..._RING_LL` p50 3.70ms ×79/前向 | **协议未变** | + +**根因：环境变量根本没到 rank 进程。** 逐一核查 `/proc//environ`：launcher（`python3 -m sglang.launch_server`）与 multiprocessing resource_tracker 均含 `NCCL_PROTO=Simple`，但**全部 8 个 scheduler rank（PP0/PP1 × TP0-3，真正跑 NCCL 的进程）都没有**——sglang 的 worker spawn 重建了环境，丢弃继承来的 NCCL 调参变量（只保留自己注入的 `NCCL_CUMEM_ENABLE=0`、`NCCL_NVLS_ENABLE=0`）。因此内核选择与时长不变是必然，e2e 差值纯属噪声。 + +**即便变量送达也救不了这个方向（带宽核算）**：96MB/3.7ms ⇒ busbw≈39GB/s ≈ PCIe Gen5 x16 实际峰值的 ~75%；若真是半带宽 LL 线格式，需要 ~78GB/s 原始速率——超 PCIe Gen5 理论上限，物理不可能。故 "_LL" 命名不代表线上跑 LL 格式（NCCL 2.28 混合 plan 命名），AR 已在近最优有效带宽上运行。协议切换的理论上限 = 把 75% 提到 ~100% ⇒ AR 3.7→2.9ms ⇒ prefill 墙钟最多 -6%。 + +**结论**：协议/算法调参作为"一行环境变量的快赢"**证伪**。留在桌面上的通信优化只有两条真路：①通信-计算重叠（把 30% 的 AR 藏进计算）；②量化/压缩 AR（把载荷减半）。若仍想穷尽协议实验，正确做法是在 rank 进程内、NCCL 初始化前设置（例如通过 deploy 已挂载的 sglang_patch 在 layer_setup.py 中 `os.environ["NCCL_PROTO"]="Simple"`），预期收益 ≤6% prefill。 + +trace 留存：`60.5:/data/hf_models/pp2tp4_profiles/E_nccl_simple/`（8 rank × 2 前向）；bench：`/data/hf_models/bs_results/nccl_simple_cc16.json`；实验部署脚本（已用后删除，命令记录于本节）：`sed 's|--ipc=host|--ipc=host -e NCCL_PROTO=Simple|' /root/deploy_par.sh`。 + +## 6. 第二轮优化实验（2026-09-07）：配置级方向全部证伪 + +按 §4 排序推进"能落地的先落地"，结果是三个方向全部被证据关闭——这本身是本轮最有价值的产出：**TP4PP2 现有配置已是本栈配置最优**，后续提升必须走开发投入。详细证据归档：sskj-review `results/analysis/optimization_round2_findings.md`。 + +### 6.1 干净 128k/90% 基线（补缺口） + +此前 D 场景的 bench 数字带 profiler 作废，TP4PP2 一直没有干净 128k 指标。原样配置补测（cc8，8×131072/512，hit 0.8999）：**TTFT p50 14.70s**（min 4.64 / max 20.02），TPOT 均值 51.7ms，输出 104.0 tok/s，e2e 均值 39.24s，retract 0。 + +### 6.2 chunk 16384 证伪（方向④关闭） + +变体（`deploy_par.sh` 第 3 位置参数）启动成功、冒烟通过，但**首个 16k 请求让全部 8 个调度器 CUDA OOM 崩溃**：flashinfer cutlass MoE 工作区随 chunk 规模增长，16384-token 前向需 3.08GB，mem-frac 0.88 下每卡仅 2.11GB 空闲。跑通需 mem-frac ≤0.85（KV 池 -5%），但物理核算表明收益主体不存在——AR 纯带宽受限（每次调用固定开销≈0，"次数减半"省不出带宽部分）、indexer 的 q×k 总功与 chunk 切分无关；剩余收益（调度/launch 摊薄）≤3%，在单轮 bench 噪声（±3%）内，却要再付 Max ITL 翻倍（混合相位单次 prefill 阻塞 1.03s→2.06s）。附带发现：prefill 分块图（backend=breakable）只捕获 ≤2048 token 档位，8192/16384 的 extend 均走 eager_runner 逐层重放图块。 + +### 6.3 AR 现成开关扫描（方向①降级为纯上游开发） + +| 开关 | 判定 | +|---|---| +| `--enable-quant-communications` | **NPU 专用**（server_args 校验非 NPU 直接 ValueError） | +| flashinfer allreduce fusion | 自动启用门控 `is_sm90 or is_sm100`——**SM120 被排除**；trtllm/mnnvl 后端为 NVLink/小 batch 设计 | +| `--enable-symm-mem` / `--enable-torch-symm-mem` | one-shot 小消息/decode 导向，96MB prefill AR 在 PCIe 不适用 | +| `--enable-nccl-nvls` | NVLink 专用 | +| `--pp-async-batch-depth` | PP 气泡仅 ~4%，非瓶颈 | + +**本栈不存在 prefill AR 重叠/量化的现成路径。** 开发落点：分块 AR 与后续 GEMM 重叠（async-TP 式 TP linear 改造）；或将 flashinfer AR 融合（fp8 量化 AR+residual+norm）扩展到 SM120 与大 token 数。 + +### 6.4 indexer 关闭（方向③：算法固有成本，修正原设想） + +- **修正 §3.3 的原始设想**："前缀 logits 增量缓存"不成立——每请求后缀 q 独有，其对共享前缀键的 logits 是该请求固有功；前缀 index-K 已由 fused_store_index_k_cache 缓存，无跨请求复用。 +- 既有 skip 仅适用 kv_len ≤ index_topk=2048（全选即正确），128k 不适用；CUDA decode 侧 skip 还被 HIP-only 门控。 +- SM120 上内核后端无选择（deepgemm 唯一，cutedsl=SM100 门控、aiter=ROCm）；模型已做 `index_topk_freq=4` 跨层 topk 共享。 +- roofline：9.37ms/次 ≈ 8192 新 token × ~122k kv × 2048 FLOP/pair（每 rank，32 头×128 维×2 ÷ TP4）≈ 2.05 TFLOP/rank → **~219 TFLOPS/rank fp8**，paged-gather 内核的合理量级，无 5-10× 调优空间。 + +结论：28% 是 DSA 在 128k 语境扫全 kv 选 top-2048 的固有成本；逃逸路径（分层/粗到细索引）是影响精度的上游研究项。 + +## 7. 复现与留存 + +``` +# 部署（获胜配置） +bash /root/deploy_par.sh "--tp 4 --pp-size 2 --disable-overlap-schedule --max-prefill-tokens 16384" nomtp + +# 采集（60.5:/root/prof_tools/） +bash profile_cap.sh # A/B/C 编排：warm_and_flush → arm → bench → wait_traces +python3 bench_128k_hit90_prof.py --concurrency 8 --num-requests 8 --run-id <新> \ + --arm-profile $BASE/D_128k --profile-steps 16 + +# 分析 +python3 pp_align.py # 跨 rank 对齐 + 每 rank 报告 + 1ms 分箱 +python3 nccl_forensics.py # AR/SendRecv 分位 + 时间聚类 +python3 census.py # 事件类别普查 +# triage 三表：skill 脚本 docker cp 进容器运行（analyze_llm_torch_profile.py triage） +``` + +- trace：`60.5:/data/hf_models/pp2tp4_profiles/{A_prefill,B_decode,D_128k}/`（8 rank × 3 场景；A 的 PP0 ~70-77MB gz） +- 分析输出：`.../pp2tp4_profiles/analysis/` +- 部署/采集日志：`/root/deploy_pp2tp4_0906*.log`、`/root/prof_cap_*.log`、`/root/phase2.log`、`/root/bs_results/cleanB.log` +- 脚本本地副本：`D:\sskj\prof_tools\`；归档：sskj-review `experiments/glm53-tp4pp2-profile-20260906/` + +## 8. 60.5 事后状态 + +- 当前运行：**TP4PP2 原样配置**（容器 glm53-nvfp4:30000，chunk 8192 / mem-frac 0.88；09-07 二轮实验后已恢复并验收 health=200）。第二轮实验证明该配置即本栈配置最优，可直接作为后续开发（PP+MTP / AR 内核）的基线。 +- 若需恢复 GLM-5.3-Flash kt 服务：`bash /root/deploy_glm53_flash_kt_gpu_graphs.sh`（恢复前 kill -9 本容器 sglang 进程并等显存归零）。 + +## 9. PP+MTP 深入（2026-09-07 第三轮，进行中） + +### 9.1 复现现场（Step 1a/1b） + +补丁原样重部署（`deploy_ppmtp_bench.sh`，7 文件挂载 + 0901 镜像，健康 ~420s）。smoke 输出与 wiki 报告字节一致（Paris. Distance … Euro.），16k 单请求 31.0s 在报告区间。cc16 压测（种子 6117）：**输入 1598 tok/s（=基线 3305.5 的 0.48×）、TPOT 133.7ms（2.4×基线）**，与 09-06 报告（0.49×/145.7ms）一致——回归稳定复现。 + +### 9.2 三源假设实测证伪 + 真凶定位（Step 1c，profiler 无 stack 12 步） + +解码轮结构（TP-0 双 stage，bs 7-8 过渡窗，12 轮 span 3.522s，**293.5ms/轮**，相位标注账目闭合）： + +| 相位 | 每轮 | 占比 | +|---|---|---| +| TARGET_VERIFY（verify 前向 39 层） | ~210ms | 72% | +| draft（3 步 eager） | ~30ms | 10% | +| DECODE 步 | ~13ms | 4% | +| draft_extend | ~5ms | 2% | +| 边界收发 + 调度 | ~35ms | 12% | + +**原三源实测全部出局**：7 处主机同步合计仅 **4.9ms/轮**（120 次 cudaStreamSynchronize 均值 0.49ms）；eager launch 256 次/轮 × 13μs ≈ **3.3ms/轮**；边界 SendRecv GPU 时长仅 **0.17ms/轮**（整卡发送在此尺度无感）。三者合计 <10ms/轮，解释不了 240ms 的回归量。 + +**真凶（两级传播链）**： + +1. **边界 GLOO 元数据通道的同步 CPU rendezvous**：`parallel_state.send_object` 每条边界字典 = 2 次 gloo isend（8B size + pickle 体），收端 `recv_object` 两次 irecv 均 `work.wait()` 真阻塞。对端迟到时发送端 gloo:send **自旋阻塞 300-540ms**（trace user_annotation 实测，~每 2 轮一次）；trace 中最大 GPU 空洞 240ms 与之对应。 +2. **TP 到达偏斜 → AllReduce 自旋**：被 gloo 卡住的 rank 在后续每个 AR 迟到，其余 3 rank 的 AR 内核空转。PP0 AR 合计 **132ms/轮（占轮时 45%、GPU busy 的 82%）**；TP2-PP0 busy 仅 30%（掉队者空闲最多），其余 rank busy 55-62%（AR 等待撑大）。 + +结论：**回归主体是协议层（边界通道停顿 + 其 TP 传播），不是执行层**。Step 3 优先级据此改写：边界通道重构（去热路径 GLOO / 预挂 recv）>> 图捕获（仅省 ~20-30ms 计算与 launch）>> 拆同步（≈0）>> 分片发送（≈0）。 + +### 9.3 竞态二分结果（Step 2，掩码热切换实测） + +工具：`eagle_worker_v2_mask.py`（7 处同步改为 `/tmp/pp_sync_mask` 位掩码热开关，默认 127 ≡ r33）+ `race_run.sh`。实验矩阵： + +| 构建 | 掩码 | 负载 | 结果 | +|---|---|---|---| +| r33 | 127 | conc_test 8×16384 | PASS 64.4s（对照，≡09-06 基线） | +| r33 | 0 | conc_test 8×16384 ×3 + 完整阶梯 | **3/3 PASS**（30.7/27.5/26.6s）+ smoke 字节一致 | +| r33 | 0 | **bench cc16 完整压测** | **CRASH**：prefill 爬坡段（16 并发+队列 churn），PG2(tp:device) NCCL watchdog 报 IMA；表面 dsa init_forward_metadata / allocator insert_events——与 09-06 记录同款 | +| r34（延迟 commit） | 0 | conc_test 8×16384 | **CRASH 23.6s**（比 r33+bench 更快） | + +三条结论： + +1. **conc_test 是不合格触发器**：8 并发杀手负载 3/3 通过给了假信心；bench（16 并发、512 输出长 decode、队列 admission churn）才触发。09-06 以全套压测为验证标准是对的。 +2. **7 处同步在 bench 剖面下确属必要遮罩**（09-06 结论维持），但其成本仅 4.9ms/轮——拆它没有收益，竞态才是封锁项。 +3. **commit 时的 work.wait() 本身也是隐性时序遮罩**：r34 把它延迟一轮后竞态反而更快触发（23.6s）——发送端节奏是竞态防护的一部分。任何"去停顿"改动（延迟 commit、预挂 recv、去 GLOO）都会扰动同一时序织物，**在竞态根因定位前一律不安全**。 + +r34 崩溃样本的表面仍为异步粘性错误（dsa metadata init 的 H2D 拷贝 + allocator insert_events），内核元凶未定位。 + +**判别实验不可行性（工具矩阵终点）**：`PYTORCH_NO_CUDA_MEMORY_CACHING=1` 部署死于启动期 CUDA graph capture（cudaErrorStreamCaptureInvalidated——capture 依赖缓存分配器的内存池）；即使能启动，cudaFree 自带的设备同步 ≈ 部分版 CUDA_LAUNCH_BLOCKING（09-06 已证其为遮罩），判别结果必然失真。结论：本竞态的根因定位只能靠 **compute-sanitizer memcheck 全 rank 包裹启动**（唯一不引入同步污染的定位器，~20min 启动成本），归入上游工程。 + +### 9.4 终局判断与上游工程清单 + +**本轮净结论**：PP+MTP 的 0.41-0.49× 回归不是"eager 无图 + 7 同步 + 4× 线路字节"的执行层问题（三者实测合计 <10ms/轮），而是**边界控制通道的协议层问题**：每轮 3 次串行 GLOO CPU rendezvous 在相位漂移下把发送端 CPU 卡死 300-540ms，传播为 TP 到达偏斜 → AR 自旋（132ms/轮）+ GPU 空洞（~130ms/轮）。同时该竞态（去遮罩即 async IMA）与停顿去除**共享同一时序织物**——7 同步与 commit wait 都是隐性屏障，任何性能手术在根因定位前都不安全。 + +上游工程清单（按依赖序）： + +1. **竞态根因定位与修复（解锁一切的前提）**：compute-sanitizer memcheck 包裹启动复现；嫌疑面已收窄到边界 P2P 生命周期/跨流序（record_stream 的流指向、`work.wait()` 事件落在 wait 时刻的当前流而消费在 forward/copy 流）。修复形态候选：P2P 专用 NCCL communicator + 显式跨流 event 协议（发送：生产流→NCCL 流；接收：NCCL 流→消费流）。 +2. **边界控制通道去 GLOO**（性能主杠杆，依赖 1）：实测停顿链见 9.2；方向 = 固定尺寸元数据走 NCCL P2P / 接收端预挂 recv（后台线程）/ 合并每轮 3 个字典为 1。 +3. **PP+spec 图捕获**（依赖 1，图回放会跳过 Python 同步）：上游 `_hosts_draft` 半成品落地后的下一步；本栈实测 prefill 分段图仅覆盖 ≤2048 token，decode 图捕获是收益 ~20-30ms/轮的第二杠杆。 +4. 已证伪/无需：拆同步（4.9ms/轮，且是必要遮罩）、分片发送（SendRecv 0.17ms/轮）、NCCL_PROTO（环回测试已关）、chunk 16384（OOM + 无收益）。 + +**60.5 终态**：TP4PP2 无 MTP 基线恢复部署（生产口径不变）。MTP 实验产物（r33/r34/掩码构建、崩溃日志、trace、脚本）留档见 §7 与 sskj-review。 diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/ppmtp_wiki_report_0906.xml b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/ppmtp_wiki_report_0906.xml new file mode 100644 index 0000000..9c395ca --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/analysis/ppmtp_wiki_report_0906.xml @@ -0,0 +1,62 @@ +

1. 背景与结论

+

上游 sglang nightly 镜像（lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729）对 GLM-5.3-NVFP4 的 TP4PP2 + EAGLE(MTP) 组合只有部分实现：非最后 PP stage 的 scheduler 缺少 draft/verify 结果的回环链路，spec 解码在 PP 下无法工作。本次在 174.1.60.5（8×RTX 6000D，容器 glm53-nvfp4）上自行补全该链路，使 TP4PP2+MTP 服务完整跑通并完成与基线同参的压测。实验全程仅在 60.5 上进行。

+ +

结论：功能层面已达成——输出正确、两 stage 副本锁步一致，楔死与崩溃分别根治/稳定化，连续负载与全套压测零故障。但当前形态下 MTP 是净性能回归：输入吞吐为 TP4PP2 无 MTP 基线的 0.41–0.49×，TPOT 为基线的 2.3–3.4×。回归不来自推测质量（accept len ≈ 2.4–2.5，接受率健康），而来自执行开销：eager draft/verify 无 CUDA 图、每轮 7 次主机同步、边界全张量发送约 4× 线路字节。三项均有明确的后续优化路径。

+ +

2. 实现内容

+

补丁以挂载文件形式注入容器（/root/sglang_patch2/，只读挂载），核心是两个文件：scheduler_pp_mixin.py（PP 边界通信协议）与 eagle_worker_v2.py（draft/verify 执行与重放）。启动方式：

+
bash /root/deploy_ppmtp.sh '--tp 4 --pp-size 2 --disable-overlap-schedule --max-prefill-tokens 16384' mtp
+# MTP 参数: --speculative-algorithm EAGLE --speculative-num-steps 3
+#           --speculative-eagle-topk 1 --speculative-num-draft-tokens 4
+

协议要点：PP0→PP1 方向由调度器把 draft/verify 所需代理张量（proxy tensors）随 batch 启动发出，经 launch_event 以 wait_event 门控，保证后级在 forward 前拿到；PP1（最后 stage）→PP0 方向把 verify 结果（hidden states、accept_lens、next_token_ids、verify_slots 等）回环给非最后 stage，由 q_event 门控；PP0 再将整理后的结果回传 PP1 供下一轮 draft。元数据先走 GLOO，数据张量再逐个走 NCCL P2P；接收端按消息类型做 typed demux（proxy/output），错序到达暂存 inbox。verify 阶段分配的 draft-KV slots 在 _forward_isolation 回滚后无法再从 batch 上读到，改为在 verify 出口显式捕获并随结果中继，非最后 stage 重放 draft-extend 时显式回填 batch.out_cache_loc。

+

调试与强制项：SGLANG_PP_SPEC_DEBUG 门控全部分布式探针日志；SGLANG_PP_SPEC_FORCE_EAGER_DRAFT=1 与 SGLANG_PP_FORCE_EAGER_VERIFY=1 强制 draft/verify 走 eager——上游没有实现 PP+spec 组合下的图捕获路径（decode 日志可见 cuda graph: False），这是性能回归的第一大头。

+

3. 两个故障的根因链

+

3.1 分块 prefill 确定性楔死（已根治）

+

现象：10684 token 的 16k 复现请求（分块 [8192, 2492]，两块）在第 2 块边界 7/7 确定性死锁，小 prompt 从不楔死。证据链：py-spy 显示 8 个 scheduler rank 全部停在自己 post-recv 的同步点、GPU 100% 空转；NCCL COLL 日志显示每个 rank 的全部 Send/Recv（各 350+ 个）实际运行在同一条 stream 上——parallel_state.send_tensor_dict/recv_tensor_dict 走的是 torch.distributed.isend/irecv（ProcessGroupNCCL），代码里 with torch.cuda.stream(...) 的包裹并不改变实际执行流。于是两端同时 send-first 时，每个 rank 的 irecv 在 FIFO 上排在自己的 isend 之后，两侧各挂一个等待对端 recv 的大发送，构成环。

+

为什么只有 spec 楔死：spec 回环携带 chunk 尺度的 hidden states（8192 token × 6144 hidden × bf16 ≈ 100MB），NCCL 对该尺寸走 rendezvous，发送必须等对端 recv 就位才完成；非 spec 与 smoke 场景的回环是小张量，走 buffered/LL 路径，发送不等对端也能完成，环不成立。

+

修复：按 pp_rank 奇偶交替排序边界 P2P——偶数 rank 先 send 后 recv，奇数 rank 先 recv 后 send，任何环形等待都被该排序解开。这条规则代码里原本就有，但只对 blocking-send 后端生效（XPU 路径注释写明），本次把它应用到 CUDA。修复后 16k 复现 3/3 通过（39.7s / 22.0s / 31.7s），同步探针收发计数 1072/1072 平衡。

+

3.2 并发负载竞态（已稳定化，底层竞态对未定位）

+

现象：debug 探针关闭后，8×16384 并发负载必然触发 async CUDA illegal memory access，崩溃表面点随机（allocator insert_events、dsa_backend init_forward_metadata 等），PP0 为首发方。定性：CUDA_LAUNCH_BLOCKING=1 下同一负载通过（78.1s），是纯竞态；消融实验证明 recv 后无条件 drain schedule stream 不能替代遮罩（36.4s 即崩），真正的遮罩是 eagle_worker_v2 中 7 个 debug 探针块的 .item()/.tolist() 主机同步。

+

修复：把这 7 处阶段边界的主机同步改为无条件执行（日志仍由 debug 门控），8×16384 复现 3/3 通过（60.7s / 30.1s / 28.1s），全套压测零崩溃。另有两项加固：NCCL Work.wait() 默认模式下是 host no-op，"提交"后即释放的张量可能在 isend/irecv 内核仍在读时被分配器复用，故对边界全部 CUDA 张量在发送侧（schedule stream + 当前流）与接收侧（forward + copy stream）补了 record_stream()；最后 stage 的发送数据门控同时 wait_event 到实际执行流（schedule_stream）。

+

诚实的限制：7 次无条件同步是遮罩不是根治——底层竞态对未定位（已排查并排除 launch 流门控顺序、分配器复用等嫌疑），定位之前不建议拆除同步。

+

4. 正确性与稳定性验证

+

正确性：标准 smoke prompt 每次运行输出字节一致（"Paris. Distance from London to Paris is 343 km, while distance from Manchester to Paris is 605 km. The currency in France is Euro."），PP0/PP1 两 stage 的 decode 日志逐行一致，副本锁步。温度 0 下随机 garbage prompt 的输出存在轻微漂移，为 batch shape 变化引入的数值非确定性（近似值 argmax 翻转），不是正确性缺陷。

+

稳定性阶梯：16k 单请求复现 3/3；conc_test 阶梯（8×64 → 8×2048 → 8×16384）全过，其中杀手负载 8×16384 为 3/3；bench_serving 全套 cc8/16/32（含 warmup 与 flush-cache）零崩溃，completed 数与请求数一致。

+

5. 压测结果

+

方法与基线完全同参：容器内 sglang.bench_serving，random-ids 数据集，输入 16384 / 输出 512、range-ratio 1.0，温度 0，--flush-cache，warmup 1，cc=8/16/32；MTP 侧种子 5301/5302/5303，基线种子 7408/7416/7432。基线取 TP4PP2（无 MTP）与 TP8/EP8 两组，详见 。

+ + + + + + + + + + + + + +
并发配置输入吞吐 (tok/s)输出吞吐 (tok/s)平均 TTFT (ms)平均 TPOT (ms)
8TP4PP2+MTP1167.6336.4918323.2127.1
8TP4PP2（基线）2829.0788.4117646.956.0
8TP8/EP82775.386.7329053.728.8
16TP4PP2+MTP1625.7950.8134744.6145.7
16TP4PP2（基线）3305.5103.329161.255.4
16TP8/EP83253.14101.6646970.854.3
32TP4PP2+MTP1692.3652.8982031.7168.3
32TP4PP2（基线）3846.42120.260479.049.6
32TP8/EP83432.52107.2780889.273.5
+

解读：MTP 侧输入吞吐为 TP4PP2 基线的 0.41–0.49×，输出吞吐同比例；TPOT 为 TP4PP2 基线的 2.3–3.4×（cc8 对 EP8 达 4.4×）；TTFT 相对受损最小，对 EP8 甚至持平或略优（cc8：18.3s vs 29.1s），说明回归集中在 decode 循环，prefill 侧的 spec 开销不大。decode 日志给出 accept len 2.48 / accept rate 0.49——每轮 verify 平均落袋 2.48 个 token，推测质量健康，回归是纯执行开销。归因排序：一是 draft/verify 全 eager 无 CUDA 图，逐层 Python 调度与 kernel launch 开销；二是每轮 7 次无条件主机同步；三是边界张量整卡发送（未按 TP 分片聚合），线路字节约为分片方案的 4 倍。

+

6. 限制与后续

+

当前实现的已知限制：

+
    +
  • 仅覆盖文本路径，mm_input_embeds 未中继，多模态输入不支持；
    • +
  • draft/verify 强制 eager（上游无 PP+spec 图捕获路径），decode 无法用图；
    • +
  • 每轮 7 次无条件主机同步是竞态遮罩，底层竞态对未定位，拆除有崩溃风险；
    • +
  • 边界张量整卡发送，线路字节约 4×；
    • +
  • SGLANG_SPEC_SKIP_ZERO_STEP_DRAFT_EXTEND 开关不支持；
    • +
  • 温度 0 下输出存在 batch-shape 级数值非确定性（见第 4 节）。
    • +
+

后续优化按收益排序：一是为 draft/verify 实现图捕获（边界 P2P 留在图外、以图间事件衔接），这是回归最大头；二是定位 3.2 节的竞态对后拆除 7 次同步；三是边界按 TP 分片发送或低精度传输压线路字节。三项落地后该组合才有与基线对齐甚至反超的空间——accept len 2.48 说明上限存在。

+

7. 产物位置

+ + + + + + + + +
产物服务器（174.1.60.5）本机（D:\sskj）
scheduler_pp_mixin.py（边界协议 + 奇偶修复 + 生命周期加固）/root/sglang_patch2/scheduler_pp_mixin_r27.py
eagle_worker_v2.py（7 处阶段边界同步 + 探针）/root/sglang_patch2/eagle_worker_v2_r33.py
部署/压测/复现脚本/root/deploy_ppmtp*.sh、/root/bench_ppmtp.sh、/root/conc_test.py、/root/smoke_repro_r25.pybench_ppmtp.sh、conc_test.py、wedge_capture.sh
压测原始 JSON（cc8/16/32）/data/hf_models/bs_results/m5_ppmtp_cc*.json（基线在 /root/bs_results/）—
diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/ops/prof_mtp_analysis.txt b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/ops/prof_mtp_analysis.txt new file mode 100644 index 0000000..7a8da02 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/results/ops/prof_mtp_analysis.txt @@ -0,0 +1,216 @@ +== pm-1788716023.1756692-TP-0-PP-0.trace.json.gz == + span 3.522s n_steps~12 round~293.5ms + GPU-busy 1939.1ms (55.1%) idle 1582.4ms (44.9%) kernels n=20106 + host cudaStreamSynchronize: n=120 total=58.6ms (per-call avg 0.49ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4164 + host cudaLaunchKernel: n=3072 cpu=39.9ms (avg 13us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=839 total=932.7ms top5=['239.5', '49.7', '46.1', '42.1', '39.1']ms + gap-us attributed to cudaStreamSynchronize overlap: 39.6ms + long host ops >1ms: n=1039 total=35837.5ms; top10: + + -21.0ms 3546.8ms [Trace] PyTorch Profiler (0) + + -8.6ms 537.4ms [user_annotation] gloo:send + + 293.5ms 517.0ms [user_annotation] gloo:send + + 275.2ms 513.9ms [user_annotation] gloo:send + + 275.2ms 513.8ms [user_annotation] gloo:send + + 2074.4ms 324.6ms [user_annotation] gloo:send + + 2074.4ms 324.6ms [user_annotation] gloo:send + + 2632.4ms 319.9ms [user_annotation] gloo:send + + 2632.4ms 319.9ms [user_annotation] gloo:send + + 3184.2ms 313.3ms [user_annotation] gloo:send + nccl_AllReduce 1588.1ms n= 1056 (45.1% span, 81.9% busy) + cutlass_moe 289.5ms n= 9612 (8.2% span, 14.9% busy) + sparse_mla 26.9ms n= 1008 (0.8% span, 1.4% busy) + other 17.5ms n= 7986 (0.5% span, 0.9% busy) + nccl_other 14.1ms n= 36 (0.4% span, 0.7% busy) + mqa_logits 3.7ms n= 204 (0.1% span, 0.2% busy) + nccl_SendRecv 2.0ms n= 204 (0.1% span, 0.1% busy) + +== pm-1788716023.1756692-TP-0-PP-1.trace.json.gz == + span 11.041s n_steps~12 round~920.1ms + GPU-busy 947.6ms (8.6%) idle 10093.5ms (91.4%) kernels n=20457 + host cudaStreamSynchronize: n=120 total=36.2ms (per-call avg 0.30ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4374 + host cudaLaunchKernel: n=3240 cpu=38.9ms (avg 12us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=1432 total=8788.7ms top5=['7772.2', '23.8', '15.3', '15.2', '14.4']ms + gap-us attributed to cudaStreamSynchronize overlap: 7870.4ms + long host ops >1ms: n=526 total=51095.8ms; top10: + + -8.3ms 11056.9ms [Trace] PyTorch Profiler (0) + + 3250.1ms 7773.9ms [user_annotation] recv_res_dict_from_prev_stage + + 3250.3ms 7770.3ms [user_annotation] gloo:recv + + 2966.7ms 283.1ms [user_annotation] gloo:send + + 2966.8ms 283.1ms [user_annotation] gloo:send + + 1305.0ms 277.6ms [user_annotation] gloo:send + + 1305.1ms 277.5ms [user_annotation] gloo:send + + 754.3ms 276.8ms [user_annotation] gloo:send + + 754.4ms 276.7ms [user_annotation] gloo:send + + 1869.2ms 275.5ms [user_annotation] gloo:send + nccl_AllReduce 535.6ms n= 1044 (4.9% span, 56.5% busy) + cutlass_moe 350.3ms n= 9876 (3.2% span, 37.0% busy) + sparse_mla 27.4ms n= 1008 (0.2% span, 2.9% busy) + other 17.5ms n= 8094 (0.2% span, 1.8% busy) + nccl_other 12.1ms n= 48 (0.1% span, 1.3% busy) + mqa_logits 3.0ms n= 180 (0.0% span, 0.3% busy) + nccl_SendRecv 2.7ms n= 207 (0.0% span, 0.3% busy) + +== pm-1788716023.1756692-TP-1-PP-0.trace.json.gz == + span 3.522s n_steps~12 round~293.5ms + GPU-busy 2179.6ms (61.9%) idle 1342.0ms (38.1%) kernels n=20106 + host cudaStreamSynchronize: n=120 total=94.4ms (per-call avg 0.79ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4164 + host cudaLaunchKernel: n=3072 cpu=37.6ms (avg 12us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=659 total=837.0ms top5=['240.0', '49.5', '45.9', '42.3', '39.6']ms + gap-us attributed to cudaStreamSynchronize overlap: 34.6ms + long host ops >1ms: n=1138 total=32034.5ms; top10: + + -17.7ms 3543.5ms [Trace] PyTorch Profiler (0) + + -8.5ms 531.8ms [user_annotation] gloo:send + + 279.8ms 507.9ms [user_annotation] gloo:send + + 279.8ms 507.9ms [user_annotation] gloo:send + + 2067.8ms 320.2ms [user_annotation] gloo:send + + 2067.8ms 320.2ms [user_annotation] gloo:send + + 2628.5ms 316.7ms [user_annotation] gloo:send + + 2628.5ms 316.7ms [user_annotation] gloo:send + + 3177.8ms 314.3ms [user_annotation] gloo:send + + 3177.8ms 314.3ms [user_annotation] gloo:send + nccl_AllReduce 1828.6ms n= 1056 (51.9% span, 83.9% busy) + cutlass_moe 288.3ms n= 9612 (8.2% span, 13.2% busy) + sparse_mla 26.7ms n= 1008 (0.8% span, 1.2% busy) + other 17.2ms n= 7986 (0.5% span, 0.8% busy) + nccl_other 16.2ms n= 36 (0.5% span, 0.7% busy) + mqa_logits 3.6ms n= 204 (0.1% span, 0.2% busy) + nccl_SendRecv 2.0ms n= 204 (0.1% span, 0.1% busy) + +== pm-1788716023.1756692-TP-1-PP-1.trace.json.gz == + span 11.046s n_steps~12 round~920.5ms + GPU-busy 2059.8ms (18.6%) idle 8986.2ms (81.4%) kernels n=20457 + host cudaStreamSynchronize: n=120 total=86.0ms (per-call avg 0.72ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4374 + host cudaLaunchKernel: n=3240 cpu=37.2ms (avg 11us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=827 total=8437.5ms top5=['7771.2', '22.5', '20.4', '15.5', '15.4']ms + gap-us attributed to cudaStreamSynchronize overlap: 7836.1ms + long host ops >1ms: n=1091 total=52267.7ms; top10: + + -8.5ms 11062.3ms [Trace] PyTorch Profiler (0) + + 3255.5ms 7772.7ms [user_annotation] recv_res_dict_from_prev_stage + + 3255.6ms 7769.1ms [user_annotation] gloo:recv + + 2965.9ms 289.2ms [user_annotation] gloo:send + + 2966.0ms 289.2ms [user_annotation] gloo:send + + 1862.4ms 287.6ms [user_annotation] gloo:send + + 1862.4ms 287.5ms [user_annotation] gloo:send + + 1302.8ms 285.2ms [user_annotation] gloo:send + + 1302.8ms 285.2ms [user_annotation] gloo:send + + 759.5ms 276.8ms [user_annotation] gloo:send + nccl_AllReduce 1638.2ms n= 1044 (14.8% span, 79.5% busy) + cutlass_moe 349.2ms n= 9876 (3.2% span, 17.0% busy) + sparse_mla 27.4ms n= 1008 (0.2% span, 1.3% busy) + nccl_other 23.7ms n= 48 (0.2% span, 1.2% busy) + other 17.7ms n= 8094 (0.2% span, 0.9% busy) + mqa_logits 3.0ms n= 180 (0.0% span, 0.1% busy) + nccl_SendRecv 2.6ms n= 207 (0.0% span, 0.1% busy) + +== pm-1788716023.1756692-TP-2-PP-0.trace.json.gz == + span 3.522s n_steps~12 round~293.5ms + GPU-busy 1056.4ms (30.0%) idle 2465.4ms (70.0%) kernels n=20106 + host cudaStreamSynchronize: n=120 total=58.8ms (per-call avg 0.49ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4164 + host cudaLaunchKernel: n=3072 cpu=41.0ms (avg 13us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=1278 total=1208.5ms top5=['240.0', '47.7', '44.1', '40.6', '37.9']ms + gap-us attributed to cudaStreamSynchronize overlap: 36.0ms + long host ops >1ms: n=621 total=31027.9ms; top10: + + -11.2ms 3537.6ms [Trace] PyTorch Profiler (0) + + 258.3ms 538.5ms [user_annotation] gloo:send + + 258.4ms 538.5ms [user_annotation] gloo:send + + -8.4ms 537.8ms [user_annotation] gloo:send + + 2077.2ms 322.5ms [user_annotation] gloo:send + + 2077.2ms 322.5ms [user_annotation] gloo:send + + 3185.7ms 319.6ms [user_annotation] gloo:send + + 3185.8ms 319.6ms [user_annotation] gloo:send + + 2635.6ms 317.1ms [user_annotation] gloo:send + + 2635.6ms 317.1ms [user_annotation] gloo:send + nccl_AllReduce 707.7ms n= 1056 (20.1% span, 67.0% busy) + cutlass_moe 289.3ms n= 9612 (8.2% span, 27.4% busy) + sparse_mla 27.0ms n= 1008 (0.8% span, 2.6% busy) + other 17.0ms n= 7986 (0.5% span, 1.6% busy) + nccl_other 10.1ms n= 36 (0.3% span, 1.0% busy) + mqa_logits 3.7ms n= 204 (0.1% span, 0.3% busy) + nccl_SendRecv 2.8ms n= 204 (0.1% span, 0.3% busy) + +== pm-1788716023.1756692-TP-2-PP-1.trace.json.gz == + span 11.040s n_steps~12 round~920.0ms + GPU-busy 2123.5ms (19.2%) idle 8916.7ms (80.8%) kernels n=20457 + host cudaStreamSynchronize: n=120 total=134.5ms (per-call avg 1.12ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4374 + host cudaLaunchKernel: n=3240 cpu=37.8ms (avg 12us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=744 total=8389.1ms top5=['7776.1', '24.2', '23.4', '20.7', '14.6']ms + gap-us attributed to cudaStreamSynchronize overlap: 7893.3ms + long host ops >1ms: n=1082 total=52177.2ms; top10: + + -8.1ms 11055.7ms [Trace] PyTorch Profiler (0) + + 3249.7ms 7777.6ms [user_annotation] recv_res_dict_from_prev_stage + + 3249.8ms 7774.0ms [user_annotation] gloo:recv + + 1869.1ms 275.0ms [user_annotation] gloo:send + + 1869.2ms 274.9ms [user_annotation] gloo:send + + 2974.5ms 274.9ms [user_annotation] gloo:send + + 2974.6ms 274.8ms [user_annotation] gloo:send + + 758.2ms 272.3ms [user_annotation] gloo:send + + 758.2ms 272.3ms [user_annotation] gloo:send + + 1312.0ms 270.1ms [user_annotation] gloo:send + nccl_AllReduce 1703.9ms n= 1044 (15.4% span, 80.2% busy) + cutlass_moe 348.0ms n= 9876 (3.2% span, 16.4% busy) + sparse_mla 27.1ms n= 1008 (0.2% span, 1.3% busy) + nccl_other 23.5ms n= 48 (0.2% span, 1.1% busy) + other 17.4ms n= 8094 (0.2% span, 0.8% busy) + mqa_logits 3.0ms n= 180 (0.0% span, 0.1% busy) + nccl_SendRecv 2.5ms n= 207 (0.0% span, 0.1% busy) + +== pm-1788716023.1756692-TP-3-PP-0.trace.json.gz == + span 3.522s n_steps~12 round~293.5ms + GPU-busy 2075.2ms (58.9%) idle 1446.3ms (41.1%) kernels n=20106 + host cudaStreamSynchronize: n=120 total=86.1ms (per-call avg 0.72ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4164 + host cudaLaunchKernel: n=3072 cpu=35.9ms (avg 12us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=806 total=908.2ms top5=['239.9', '49.7', '45.9', '42.3', '39.7']ms + gap-us attributed to cudaStreamSynchronize overlap: 34.7ms + long host ops >1ms: n=1110 total=31972.9ms; top10: + + -30.9ms 3556.9ms [Trace] PyTorch Profiler (0) + + -8.6ms 538.8ms [user_annotation] gloo:send + + 273.2ms 516.4ms [user_annotation] gloo:send + + 273.3ms 516.4ms [user_annotation] gloo:send + + 2067.2ms 325.5ms [user_annotation] gloo:send + + 2067.2ms 325.5ms [user_annotation] gloo:send + + 2625.7ms 323.3ms [user_annotation] gloo:send + + 2625.7ms 323.2ms [user_annotation] gloo:send + + 3177.4ms 316.5ms [user_annotation] gloo:send + + 3177.4ms 316.5ms [user_annotation] gloo:send + nccl_AllReduce 1726.3ms n= 1056 (49.0% span, 83.2% busy) + cutlass_moe 289.0ms n= 9612 (8.2% span, 13.9% busy) + sparse_mla 26.8ms n= 1008 (0.8% span, 1.3% busy) + other 17.4ms n= 7986 (0.5% span, 0.8% busy) + nccl_other 13.1ms n= 36 (0.4% span, 0.6% busy) + mqa_logits 3.7ms n= 204 (0.1% span, 0.2% busy) + nccl_SendRecv 2.0ms n= 204 (0.1% span, 0.1% busy) + +== pm-1788716023.1756692-TP-3-PP-1.trace.json.gz == + span 11.040s n_steps~12 round~920.0ms + GPU-busy 2171.9ms (19.7%) idle 8868.2ms (80.3%) kernels n=20457 + host cudaStreamSynchronize: n=120 total=111.6ms (per-call avg 0.93ms) + host cudaEventSynchronize: n=12 total=0.1ms cudaEventRecord n=4374 + host cudaLaunchKernel: n=3240 cpu=38.8ms (avg 12us) memcpy n=314 gpu=0.3ms + gaps>0.3ms: n=692 total=8361.0ms top5=['7770.1', '21.1', '16.9', '15.9', '14.1']ms + gap-us attributed to cudaStreamSynchronize overlap: 7846.8ms + long host ops >1ms: n=1083 total=52334.1ms; top10: + + -7.5ms 11055.2ms [Trace] PyTorch Profiler (0) + + 3249.5ms 7771.5ms [user_annotation] recv_res_dict_from_prev_stage + + 3249.7ms 7768.0ms [user_annotation] gloo:recv + + 2961.9ms 287.2ms [user_annotation] gloo:send + + 2962.0ms 287.2ms [user_annotation] gloo:send + + 1298.2ms 283.7ms [user_annotation] gloo:send + + 1298.2ms 283.7ms [user_annotation] gloo:send + + 1860.6ms 283.3ms [user_annotation] gloo:send + + 1860.7ms 283.3ms [user_annotation] gloo:send + + 753.5ms 276.9ms [user_annotation] gloo:send + nccl_AllReduce 1753.4ms n= 1044 (15.9% span, 80.7% busy) + cutlass_moe 348.1ms n= 9876 (3.2% span, 16.0% busy) + sparse_mla 27.1ms n= 1008 (0.2% span, 1.2% busy) + nccl_other 22.3ms n= 48 (0.2% span, 1.0% busy) + other 17.5ms n= 8094 (0.2% span, 0.8% busy) + mqa_logits 3.0ms n= 180 (0.0% span, 0.1% busy) + nccl_SendRecv 2.4ms n= 207 (0.0% span, 0.1% busy) + diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_mask.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_mask.sh new file mode 100644 index 0000000..b654891 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_mask.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# PP+MTP 竞态二分实验部署（60.5, nightly-dev-cu13-20260901-07c8f729） +# 与 deploy_ppmtp_bench.sh 唯一差异：eagle_worker_v2 挂载掩码版（eagle_worker_v2_mask.py）。 +# 同步掩码运行时读 /tmp/pp_sync_mask（16进制或10进制，缺省=127 全同步）， +# 可在容器外热改: docker exec glm53-nvfp4 sh -c 'echo 0x40 > /tmp/pp_sync_mask' +# 用法: bash deploy_ppmtp_mask.sh "<并行参数>" [mtp|nomtp] [chunk] [memfrac] +PAR=${1:?usage: deploy_ppmtp_mask.sh "" [mtp|nomtp] [chunk] [memfrac]} +MTPMODE=${2:-nomtp} +CHUNK=${3:-8192} +MEMFRAC=${4:-0.88} + +MTPARGS="" +if [ "$MTPMODE" = "mtp" ]; then + MTPARGS="--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4" +fi + +docker update --restart=no glm53-nvfp4 >/dev/null 2>&1 +for i in 1 2 3 4 5; do + docker rm -f glm53-nvfp4 >/dev/null 2>&1 + sleep 2 + docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$' || break +done +if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then + echo "ERROR: old container cannot be removed"; exit 1 +fi +for i in $(seq 1 15); do ss -ltn 2>/dev/null | grep -q ":30000 " || break; sleep 2; done + +docker run -d --name glm53-nvfp4 --gpus all --shm-size 64g --ipc=host --cap-add SYS_PTRACE \ + -v /root/sglang_patch2/layer_setup.py:/sgl-workspace/sglang/python/sglang/srt/model_executor/model_runner_components/layer_setup.py:ro \ + -v /root/sglang_patch2/validation_hook.py:/sgl-workspace/sglang/python/sglang/srt/arg_groups/validation_hook.py:ro \ + -v /root/eagle_worker_v2_mask.py:/sgl-workspace/sglang/python/sglang/srt/speculative/eagle_worker_v2.py:ro \ + -v /root/sglang_patch2/eagle_worker_common.py:/sgl-workspace/sglang/python/sglang/srt/speculative/eagle_worker_common.py:ro \ + -v /root/sglang_patch2/deepseek_nextn.py:/sgl-workspace/sglang/python/sglang/srt/models/deepseek_nextn.py:ro \ + -v /root/sglang_patch2/scheduler_pp_mixin.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler_pp_mixin.py:ro \ + -v /root/sglang_patch2/deepseek_v2.py:/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py:ro \ + -e SGLANG_PP_SPEC_DEBUG=0 \ + -e SGLANG_PP_SPEC_FORCE_EAGER_DRAFT=1 \ + -e SGLANG_PP_FORCE_EAGER_VERIFY=1 \ + --restart no -p 30000:30000 \ + -v /data/hf_models:/data/hf_models \ + lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 \ + python3 -m sglang.launch_server \ + --model-path /data/hf_models/GLM-5.3-NVFP4 \ + --tp 8 \ + --mem-fraction-static ${MEMFRAC} \ + --max-running-requests 16 \ + --chunked-prefill-size ${CHUNK} \ + --disable-shared-experts-fusion \ + --moe-runner-backend flashinfer_cutlass \ + --disable-flashinfer-autotune \ + --reasoning-parser glm45 --tool-call-parser glm47 \ + --enable-hierarchical-cache --hicache-ratio 3 \ + ${MTPARGS} \ + ${PAR} \ + --host 0.0.0.0 --port 30000 + +echo "deployed: par=[${PAR}] mtp=${MTPMODE} mask-file=/tmp/pp_sync_mask (default 127); waiting for health..." +for i in $(seq 10 10 1800); do + code=$(curl -s -o /dev/null -m3 -w '%{http_code}' http://127.0.0.1:30000/health 2>/dev/null) + if [ "$code" = "200" ]; then + echo "healthy after ${i}s" + docker logs glm53-nvfp4 2>&1 | grep -oE "max_total_num_tokens = [0-9]+" | head -1 + exit 0 + fi + if ! docker ps --format '{{.Names}}' | grep -q '^glm53-nvfp4$'; then + echo "CONTAINER DIED after ${i}s"; docker logs --tail 60 glm53-nvfp4 2>&1 | grep -iE "error|assert|not support|incompatible" | tail -8; exit 1 + fi + sleep 10 +done +echo "TIMEOUT waiting for health"; exit 1 diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34.sh new file mode 100644 index 0000000..9e00d5f --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# PP+MTP 竞态二分实验部署（60.5, nightly-dev-cu13-20260901-07c8f729） +# 与 deploy_ppmtp_bench.sh 唯一差异：eagle_worker_v2 挂载掩码版（eagle_worker_v2_mask.py）。 +# 同步掩码运行时读 /tmp/pp_sync_mask（16进制或10进制，缺省=127 全同步）， +# 可在容器外热改: docker exec glm53-nvfp4 sh -c 'echo 0x40 > /tmp/pp_sync_mask' +# 用法: bash deploy_ppmtp_mask.sh "<并行参数>" [mtp|nomtp] [chunk] [memfrac] +PAR=${1:?usage: deploy_ppmtp_mask.sh "" [mtp|nomtp] [chunk] [memfrac]} +MTPMODE=${2:-nomtp} +CHUNK=${3:-8192} +MEMFRAC=${4:-0.88} + +MTPARGS="" +if [ "$MTPMODE" = "mtp" ]; then + MTPARGS="--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4" +fi + +docker update --restart=no glm53-nvfp4 >/dev/null 2>&1 +for i in 1 2 3 4 5; do + docker rm -f glm53-nvfp4 >/dev/null 2>&1 + sleep 2 + docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$' || break +done +if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then + echo "ERROR: old container cannot be removed"; exit 1 +fi +for i in $(seq 1 15); do ss -ltn 2>/dev/null | grep -q ":30000 " || break; sleep 2; done + +docker run -d --name glm53-nvfp4 --gpus all --shm-size 64g --ipc=host --cap-add SYS_PTRACE \ + -v /root/sglang_patch2/layer_setup.py:/sgl-workspace/sglang/python/sglang/srt/model_executor/model_runner_components/layer_setup.py:ro \ + -v /root/sglang_patch2/validation_hook.py:/sgl-workspace/sglang/python/sglang/srt/arg_groups/validation_hook.py:ro \ + -v /root/eagle_worker_v2_mask.py:/sgl-workspace/sglang/python/sglang/srt/speculative/eagle_worker_v2.py:ro \ + -v /root/sglang_patch2/eagle_worker_common.py:/sgl-workspace/sglang/python/sglang/srt/speculative/eagle_worker_common.py:ro \ + -v /root/sglang_patch2/deepseek_nextn.py:/sgl-workspace/sglang/python/sglang/srt/models/deepseek_nextn.py:ro \ + -v /root/scheduler_pp_mixin_r34.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler_pp_mixin.py:ro \ + -v /root/sglang_patch2/deepseek_v2.py:/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py:ro \ + -e SGLANG_PP_SPEC_DEBUG=0 \ + -e SGLANG_PP_SPEC_FORCE_EAGER_DRAFT=1 \ + -e SGLANG_PP_FORCE_EAGER_VERIFY=1 \ + --restart no -p 30000:30000 \ + -v /data/hf_models:/data/hf_models \ + lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 \ + python3 -m sglang.launch_server \ + --model-path /data/hf_models/GLM-5.3-NVFP4 \ + --tp 8 \ + --mem-fraction-static ${MEMFRAC} \ + --max-running-requests 16 \ + --chunked-prefill-size ${CHUNK} \ + --disable-shared-experts-fusion \ + --moe-runner-backend flashinfer_cutlass \ + --disable-flashinfer-autotune \ + --reasoning-parser glm45 --tool-call-parser glm47 \ + --enable-hierarchical-cache --hicache-ratio 3 \ + ${MTPARGS} \ + ${PAR} \ + --host 0.0.0.0 --port 30000 + +echo "deployed: par=[${PAR}] mtp=${MTPMODE} mask-file=/tmp/pp_sync_mask (default 127); waiting for health..." +for i in $(seq 10 10 1800); do + code=$(curl -s -o /dev/null -m3 -w '%{http_code}' http://127.0.0.1:30000/health 2>/dev/null) + if [ "$code" = "200" ]; then + echo "healthy after ${i}s"; docker exec glm53-nvfp4 sh -c "echo ${SYNCMASK:-127} > /tmp/pp_sync_mask" + docker logs glm53-nvfp4 2>&1 | grep -oE "max_total_num_tokens = [0-9]+" | head -1 + exit 0 + fi + if ! docker ps --format '{{.Names}}' | grep -q '^glm53-nvfp4$'; then + echo "CONTAINER DIED after ${i}s"; docker logs --tail 60 glm53-nvfp4 2>&1 | grep -iE "error|assert|not support|incompatible" | tail -8; exit 1 + fi + sleep 10 +done +echo "TIMEOUT waiting for health"; exit 1 diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34_nocache.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34_nocache.sh new file mode 100644 index 0000000..1df432f --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/deploy_ppmtp_r34_nocache.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# PP+MTP 竞态二分实验部署（60.5, nightly-dev-cu13-20260901-07c8f729） +# 与 deploy_ppmtp_bench.sh 唯一差异：eagle_worker_v2 挂载掩码版（eagle_worker_v2_mask.py）。 +# 同步掩码运行时读 /tmp/pp_sync_mask（16进制或10进制，缺省=127 全同步）， +# 可在容器外热改: docker exec glm53-nvfp4 sh -c 'echo 0x40 > /tmp/pp_sync_mask' +# 用法: bash deploy_ppmtp_mask.sh "<并行参数>" [mtp|nomtp] [chunk] [memfrac] +PAR=${1:?usage: deploy_ppmtp_mask.sh "" [mtp|nomtp] [chunk] [memfrac]} +MTPMODE=${2:-nomtp} +CHUNK=${3:-8192} +MEMFRAC=${4:-0.88} + +MTPARGS="" +if [ "$MTPMODE" = "mtp" ]; then + MTPARGS="--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4" +fi + +docker update --restart=no glm53-nvfp4 >/dev/null 2>&1 +for i in 1 2 3 4 5; do + docker rm -f glm53-nvfp4 >/dev/null 2>&1 + sleep 2 + docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$' || break +done +if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then + echo "ERROR: old container cannot be removed"; exit 1 +fi +for i in $(seq 1 15); do ss -ltn 2>/dev/null | grep -q ":30000 " || break; sleep 2; done + +docker run -d --name glm53-nvfp4 --gpus all --shm-size 64g --ipc=host --cap-add SYS_PTRACE \ + -v /root/sglang_patch2/layer_setup.py:/sgl-workspace/sglang/python/sglang/srt/model_executor/model_runner_components/layer_setup.py:ro \ + -v /root/sglang_patch2/validation_hook.py:/sgl-workspace/sglang/python/sglang/srt/arg_groups/validation_hook.py:ro \ + -v /root/eagle_worker_v2_mask.py:/sgl-workspace/sglang/python/sglang/srt/speculative/eagle_worker_v2.py:ro \ + -v /root/sglang_patch2/eagle_worker_common.py:/sgl-workspace/sglang/python/sglang/srt/speculative/eagle_worker_common.py:ro \ + -v /root/sglang_patch2/deepseek_nextn.py:/sgl-workspace/sglang/python/sglang/srt/models/deepseek_nextn.py:ro \ + -v /root/scheduler_pp_mixin_r34.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler_pp_mixin.py:ro \ + -v /root/sglang_patch2/deepseek_v2.py:/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py:ro \ + -e SGLANG_PP_SPEC_DEBUG=0 \ + -e SGLANG_PP_SPEC_FORCE_EAGER_DRAFT=1 \ + -e SGLANG_PP_FORCE_EAGER_VERIFY=1 -e PYTORCH_NO_CUDA_MEMORY_CACHING=1 \ + --restart no -p 30000:30000 \ + -v /data/hf_models:/data/hf_models \ + lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 \ + python3 -m sglang.launch_server \ + --model-path /data/hf_models/GLM-5.3-NVFP4 \ + --tp 8 \ + --mem-fraction-static ${MEMFRAC} \ + --max-running-requests 16 \ + --chunked-prefill-size ${CHUNK} \ + --disable-shared-experts-fusion \ + --moe-runner-backend flashinfer_cutlass \ + --disable-flashinfer-autotune \ + --reasoning-parser glm45 --tool-call-parser glm47 \ + --enable-hierarchical-cache --hicache-ratio 3 \ + ${MTPARGS} \ + ${PAR} \ + --host 0.0.0.0 --port 30000 + +echo "deployed: par=[${PAR}] mtp=${MTPMODE} mask-file=/tmp/pp_sync_mask (default 127); waiting for health..." +for i in $(seq 10 10 1800); do + code=$(curl -s -o /dev/null -m3 -w '%{http_code}' http://127.0.0.1:30000/health 2>/dev/null) + if [ "$code" = "200" ]; then + echo "healthy after ${i}s"; docker exec glm53-nvfp4 sh -c "echo ${SYNCMASK:-127} > /tmp/pp_sync_mask" + docker logs glm53-nvfp4 2>&1 | grep -oE "max_total_num_tokens = [0-9]+" | head -1 + exit 0 + fi + if ! docker ps --format '{{.Names}}' | grep -q '^glm53-nvfp4$'; then + echo "CONTAINER DIED after ${i}s"; docker logs --tail 60 glm53-nvfp4 2>&1 | grep -iE "error|assert|not support|incompatible" | tail -8; exit 1 + fi + sleep 10 +done +echo "TIMEOUT waiting for health"; exit 1 diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/eagle_worker_v2_mask.py b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/eagle_worker_v2_mask.py new file mode 100644 index 0000000..765b55c --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/eagle_worker_v2_mask.py @@ -0,0 +1,1962 @@ +import contextlib +import logging +import os +import time +from dataclasses import replace +from typing import List, Optional + +import torch + +from sglang.kernels.ops.speculative.topk1 import draft_topk1_postprocess +from sglang.srt.distributed import get_pp_group +from sglang.srt.distributed.parallel_state_wrapper import ParallelState +from sglang.srt.environ import envs +from sglang.srt.hardware_backend.npu.graph_runner.eagle_draft_extend_npu_graph_runner import ( + EAGLEDraftExtendNpuGraphRunner, +) +from sglang.srt.hardware_backend.npu.graph_runner.eagle_draft_npu_graph_runner import ( + EAGLEDraftNpuGraphRunner, +) +from sglang.srt.hardware_backend.npu.graph_runner.npu_graph_runner import NPUGraphRunner +from sglang.srt.kv_canary.runner.canary_manager import context_tuple +from sglang.srt.layers.attention.flashinfer_backend import FlashInferAttnBackend +from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState +from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend +from sglang.srt.layers.attention.triton_backend import TritonAttnBackend +from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend +from sglang.srt.layers.attention.trtllm_mla_backend import ( + TRTLLMMLABackend, +) +from sglang.srt.layers.moe.utils import ( + draft_model_build_scope, + speculative_moe_a2a_backend_context, + speculative_moe_backend_context, +) +from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput +from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.managers.scheduler import GenerationBatchResult +from sglang.srt.managers.tp_worker import TpModelWorker +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + check_cuda_graph_backend, +) +from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardBatch +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context +from sglang.srt.model_executor.runner import ( + DecodeCudaGraphRunner, + get_batch_sizes_to_capture, +) +from sglang.srt.runtime_context import ( + get_context, + get_device, + get_exec, + get_model, + get_parallel, + get_schedule, + get_spec, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.speculative.adaptive_runtime_state import ( + AdaptiveController, + SpecRuntimeState, +) +from sglang.srt.speculative.base_spec_worker import BaseSpecWorker, EagleDraftWorkerBase +from sglang.srt.speculative.draft_utils import DraftBackendFactory +from sglang.srt.speculative.eagle_draft_cuda_graph_runner import ( + EAGLEDraftCudaGraphRunner, +) +from sglang.srt.speculative.eagle_draft_extend_cuda_graph_runner import ( + EAGLEDraftExtendCudaGraphRunner, +) +from sglang.srt.speculative.eagle_info import ( + EagleDraftExtendInput, + EagleDraftInput, + EagleVerifyInput, +) +from sglang.srt.speculative.eagle_utils import ( + _eagle_prefill_tail_tokens, + default_tree_mask_mode, + get_draft_recurrent_hidden_state_spec, + organize_draft_results, + per_step_draft_out_cache_loc, +) +from sglang.srt.speculative.eagle_worker_common import ( + build_eagle_verify_input, + prepare_for_draft, + prepare_for_draft_extend, + run_eagle_verify, +) +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.srt.speculative.spec_utils import ( + draft_pp_context, + draft_tp_context, + fast_sample, + get_plan_stream, + load_token_map, + renorm_draft_probs, + sample_draft_proposal, + select_top_k_tokens, + spec_stage_span, +) +from sglang.srt.utils.async_probe import ( + maybe_detect_inf, + maybe_detect_nan, + maybe_detect_oob, +) +from sglang.srt.utils.common import ( + MultiprocessingSerializer, + empty_context, + fast_topk, + get_available_gpu_memory, + is_cpu, + is_cuda, + is_hip, + is_musa, + is_npu, + is_xpu, + log_info_on_rank0, +) +from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions + +_is_cpu = is_cpu() +_is_npu = is_npu() +_is_cuda = is_cuda() +_is_musa = is_musa() +_is_hip = is_hip() +_is_xpu = is_xpu() + + +logger = logging.getLogger(__name__) + +# [EXPERIMENT] PP+spec debug: per-round cross-stage draft-tree fingerprints +# and acceptance stats. Enable with SGLANG_PP_SPEC_DEBUG=1 and diff [PPSPEC] +# lines between PP stages — diverging tree_sum means replica drift. +_PP_SPEC_DEBUG = os.getenv("SGLANG_PP_SPEC_DEBUG", "0") == "1" + +# [EXPERIMENT] PP+spec race-bisect: bit k-1 toggles stage-boundary sync site k. +# bit0(1) site1 draftstep (post draft_forward, pre build_eagle_verify_input) +# bit1(2) site2 prefill-ext (post fast_topk, pre next-iter draft spec_info) +# bit2(4) site3 ext-pre (pre _draft_extend_for_decode forward) +# bit3(8) site4 extlogits (post draft-extend forward, pre detect_nan/publish) +# bit4(16) site5 draft-tree (post draft(), pre verify()) +# bit5(32) site6 accept (post _draft_extend_for_decode, hosts_draft only) +# bit6(64) site7 replay-in (PP0 ring replay, pre _draft_extend_for_decode) +# Default 0x7F = all 7 unconditional syncs active (= shipped race mask). +# The per-call check prefers /tmp/pp_sync_mask (hot-editable between bisect +# runs without redeploying); env is the import-time fallback. +_PP_SYNC_MASK_ENV = os.getenv("SGLANG_PP_SPEC_SYNC_MASK", "127") + + +def _pp_sync_mask() -> int: + try: + with open("/tmp/pp_sync_mask") as _f: + return int(_f.read().strip(), 0) + except Exception: + return int(_PP_SYNC_MASK_ENV, 0) + + +class EagleDraftWorker(EagleDraftWorkerBase): + def __init__( + self, + server_args: ServerArgs, + gpu_id: int, + ps: ParallelState, + nccl_port: int, + target_worker: TpModelWorker, + ): + super().__init__() + + # copy args + self.server_args = server_args + self.gpu_id = gpu_id + self.ps = ps + self.nccl_port = nccl_port + self.target_worker = target_worker + + # Args for easy access + self.device = get_device().device + self.topk = get_spec().speculative_eagle_topk + if get_spec().speculative_use_rejection_sampling: + assert self.topk == 1, "Chain speculative sampling supports only topk=1" + self.speculative_num_steps = get_spec().speculative_num_steps + self.speculative_num_draft_tokens = get_spec().speculative_num_draft_tokens + self.speculative_algorithm = SpeculativeAlgorithm.from_string( + get_spec().speculative_algorithm + ) + + self._rebuild_topk1_chain_buffers() + + # Load draft model weights only. + if ( + get_parallel().enable_dp_attention + and self.speculative_algorithm.is_eagle3() + ): + ctx = draft_tp_context(get_parallel().attn_tp_group) + else: + ctx = empty_context() + with ( + ctx + ), draft_pp_context(), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), draft_model_build_scope(): + self.draft_worker = TpModelWorker( + server_args=server_args, + gpu_id=gpu_id, + # spec workers don't support pipeline parallelism + ps=replace(ps, pp_rank=0, pp_size=1), + nccl_port=nccl_port, + is_draft_worker=True, + # The draft runs at absolute target positions. + context_length=target_worker.model_runner.model_config.context_len, + random_seed=target_worker.random_seed, + ) + + # Alias for better readability + self.draft_runner = self.draft_worker.model_runner + self._init_dsa_index_share_state() + # Eager draft-extend seed buffer (graph paths use their own static ones). + self.dsa_extend_topk_buf: Optional[torch.Tensor] = None + self.draft_tp_context = ( + draft_tp_context if get_parallel().enable_dp_attention else empty_context + ) + self.tree_mask_mode = default_tree_mask_mode() + + self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) + + def alloc_memory_pool( + self, + memory_pool_config=None, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + ): + """Allocate draft KV cache pools (called by scheduler).""" + self.req_to_token_pool = req_to_token_pool + self.token_to_kv_pool_allocator = token_to_kv_pool_allocator + self.draft_worker.alloc_memory_pool( + memory_pool_config=memory_pool_config, + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=token_to_kv_pool_allocator, + ) + self.init_token_map() + self.init_lm_head() + + if get_spec().speculative_use_rejection_sampling: + target_vocab_size = self.target_worker.model_config.vocab_size + draft_vocab_size = ( + self.hot_token_id.shape[0] + if self.hot_token_id is not None + else target_vocab_size + ) + # FIXME: support reduced (hot) draft vocab by scattering draft probs + # into the target vocab via the d2t map before the sampling kernel. + if draft_vocab_size != target_vocab_size: + raise ValueError( + "--speculative-use-rejection-sampling requires the draft and " + f"target to share one vocab, but the draft vocab " + f"({draft_vocab_size}) != target vocab ({target_vocab_size})." + ) + + def init_attention_backends(self): + with ( + self.draft_tp_context(self.draft_runner.tp_group), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + ): + self.draft_worker.init_attention_backends() + self.init_attention_backend() + + def init_cuda_graphs(self): + with ( + self.draft_tp_context(self.draft_runner.tp_group), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + ): + self.draft_worker.init_cuda_graphs(capture_decode_cuda_graph=False) + if check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE): + self.draft_runner.init_prefill_cuda_graph(force_for_draft_worker=True) + self._capture_cuda_graphs() + + if (c := self.draft_runner.canary_manager) is not None: + c.mark_init_finished() + + def _init_dsa_index_share_state(self) -> None: + # Populate DSA index-share fields from the draft runner's hf_config. + # Reused by the attention unit-test harnesses, which skip __init__. + hf_config = self.draft_runner.model_config.hf_config + # Reuse the first draft step's DSA indexer topk across the rest; + # topk == 1 only (select_top_k_tokens reorders rows, desyncing indices). + self.index_share_for_mtp_iteration = ( + getattr(hf_config, "index_share_for_mtp_iteration", False) + and self.topk == 1 + ) + # GLM-5.2 MTP IndexShare: seed reused indexer top-k from draft-extend + # (last verified token), not draft-decode step 0. + self.dsa_index_topk = getattr(hf_config, "index_topk", None) + self.seed_dsa_topk_from_draft_extend = ( + self.index_share_for_mtp_iteration and self.dsa_index_topk is not None + ) + + def init_token_map(self): + # Load hot token ids + if self.speculative_algorithm.is_eagle3(): + if get_spec().speculative_token_map is not None: + logger.warning( + "Speculative token map specified, but EAGLE3 models already have this. Ignoring the specified token map." + ) + self.hot_token_id = None + elif get_spec().speculative_token_map is not None: + self.hot_token_id = load_token_map(get_spec().speculative_token_map) + else: + self.hot_token_id = None + + def init_lm_head(self): + from sglang.srt.lora.layers import unwrap_lora_layer + from sglang.srt.layers.utils.common import PPMissingLayer + + target_model = self.target_worker.model_runner.model + tgt_embed = getattr(target_model.model, "embed_tokens", None) + tgt_head = getattr(target_model, "lm_head", None) + embed_missing = isinstance(tgt_embed, PPMissingLayer) or tgt_embed is None + head_missing = isinstance(tgt_head, PPMissingLayer) or tgt_head is None + if embed_missing or head_missing: + # [EXPERIMENT] PP+MTP: under PP the target's embed_tokens and + # lm_head live on different stages, so the upstream whole-model + # share cannot run. But the NextN loader intentionally SKIPS the + # top-level embed/lm_head names (they are normally donated by + # the target), so skipping the share entirely leaves the draft + # with zero weights. Share what the local target stage has and + # backfill the rest from the draft checkpoint. + draft_model = self.draft_runner.model + if not embed_missing: + del draft_model.model.embed_tokens.weight + draft_model.model.embed_tokens.weight = tgt_embed.weight + if not head_missing: + del draft_model.lm_head.weight + draft_model.lm_head.weight = tgt_head.weight + if embed_missing: + self._backfill_draft_vocab_weight( + "model.embed_tokens.weight", draft_model.model.embed_tokens + ) + if head_missing: + self._backfill_draft_vocab_weight( + "lm_head.weight", draft_model.lm_head + ) + return + embed, head = target_model.get_embed_and_head() + target_lm_head = unwrap_lora_layer( + getattr(target_model, "lm_head", None) + ) + + def maybe_share_target_lm_head(): + if ( + target_lm_head is not None + and self.hot_token_id is None + and getattr(self.draft_runner.model, "hot_token_id", None) is None + and hasattr(self.draft_runner.model, "set_lm_head_from_target") + ): + self.draft_runner.model.set_lm_head_from_target(target_lm_head) + + if self.speculative_algorithm.is_eagle3(): + # most cases EAGLE3 models don't share lm_head + # but some models (e.g. nvidia/gpt-oss-120b-Eagle3) shares + if ( + hasattr(self.draft_runner.model, "load_lm_head_from_target") + and self.draft_runner.model.load_lm_head_from_target + ): + self.draft_runner.model.set_embed_and_head(embed, head) + maybe_share_target_lm_head() + else: + self.draft_runner.model.set_embed(embed) + + # grab hot token ids + if self.draft_runner.model.hot_token_id is not None: + self.hot_token_id = self.draft_runner.model.hot_token_id.to( + embed.device + ) + + else: + if self.hot_token_id is not None and head is not None: + head = head.clone() + self.hot_token_id = self.hot_token_id.to(head.device) + head.data = head.data[self.hot_token_id] + + # Share the embedding and lm_head + self.draft_runner.model.set_embed_and_head(embed, head) + maybe_share_target_lm_head() + + def _backfill_draft_vocab_weight(self, ckpt_name: str, module): + """[EXPERIMENT] PP+MTP: load a vocab-parallel weight (embed_tokens / + lm_head) straight from the draft checkpoint when the local target + stage cannot donate it — the other PP stage hosts its copy, and the + NextN loader never loads these names itself.""" + import json + + from safetensors import safe_open + + model_path = ( + get_spec().speculative_draft_model_path or self.server_args.model_path + ) + with open(os.path.join(model_path, "model.safetensors.index.json")) as f: + weight_map = json.load(f)["weight_map"] + file_name = weight_map[ckpt_name] + with safe_open(os.path.join(model_path, file_name), framework="pt") as f: + weight = f.get_tensor(ckpt_name) + shard = ( + weight.chunk(self.ps.tp_size, dim=0)[self.ps.tp_rank] + .to(self.device) + .contiguous() + ) + assert shard.shape == tuple(module.weight.shape), ( + f"[PPSPEC] backfill {ckpt_name}: checkpoint shard " + f"{tuple(shard.shape)} != module weight {tuple(module.weight.shape)} " + f"(TP sharding mismatch)" + ) + del module.weight + module.weight = torch.nn.Parameter(shard, requires_grad=False) + logger.info( + f"[PPSPEC] backfilled draft {ckpt_name} from checkpoint: " + f"shape={tuple(shard.shape)} absmax={shard.abs().max().item():.3e}" + ) + + def init_attention_backend(self): + # Create multi-step attn backends and cuda graph runners + + self.draft_extend_attn_backend = None + + draft_backend_factory = DraftBackendFactory( + self.draft_runner, + self.topk, + self.speculative_num_steps, + seed_dsa_topk_from_draft_extend=self.seed_dsa_topk_from_draft_extend, + ) + + # Initialize decode attention backend + self.draft_attn_backend = draft_backend_factory.create_decode_backend() + + # Initialize draft extend attention backend (respects speculative_attention_mode setting) + self.draft_extend_attn_backend = ( + draft_backend_factory.create_draft_extend_backend() + ) + + self.draft_runner.draft_attn_backend = self.draft_attn_backend + if self.draft_extend_attn_backend is not None: + self.draft_runner.attn_backend = self.draft_extend_attn_backend + self.tree_mask_mode = default_tree_mask_mode() + + def _capture_cuda_graphs(self): + """Capture the draft worker's own cuda graphs (decode + draft-extend).""" + self.cuda_graph_runner = None + self.cuda_graph_runner_for_draft_extend = None + + if _is_cpu or check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED): + return + + if get_model().model_impl == "mindspore": + return + + Device2DraftCudaGraphRunner = { + "xpu": EAGLEDraftCudaGraphRunner, + "npu": EAGLEDraftNpuGraphRunner, + "cuda": EAGLEDraftCudaGraphRunner, + "musa": EAGLEDraftCudaGraphRunner, + } + # Capture draft + decode_backend = get_exec().graph.cuda_graph_config.decode.backend + capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner) + if self.speculative_num_steps > 1: + tic = time.perf_counter() + before_mem = get_available_gpu_memory(self.device, self.gpu_id) + log_info_on_rank0( + logger, + f"Capture draft decode CUDA graph begin. backend={decode_backend}, " + f"num_tokens_per_req={self.topk}, bs={capture_bs}, " + f"avail mem={before_mem:.2f} GB", + ) + self.cuda_graph_runner = Device2DraftCudaGraphRunner[ + self.target_worker.device + ](self) + after_mem = get_available_gpu_memory(self.device, self.gpu_id) + capture_time = time.perf_counter() - tic + self._specialized_graph_memory_usage["draft_decode"] = ( + self._specialized_graph_memory_usage.get("draft_decode", 0.0) + + before_mem + - after_mem + ) + self._specialized_graph_time_usage["draft_decode"] = ( + self._specialized_graph_time_usage.get("draft_decode", 0.0) + + capture_time + ) + log_info_on_rank0( + logger, + "Capture draft decode CUDA graph end. " + f"elapsed={capture_time:.2f} s, " + f"mem usage={(before_mem - after_mem):.2f} GB, " + f"avail mem={after_mem:.2f} GB.", + ) + + Device2ExtendCudaGraphRunner = { + "xpu": EAGLEDraftExtendCudaGraphRunner, + "npu": EAGLEDraftExtendNpuGraphRunner, + "cuda": EAGLEDraftExtendCudaGraphRunner, + "musa": EAGLEDraftCudaGraphRunner, + } + supports_hip_draft_extend_graph = False + if _is_hip: + # Keep imports local so non-HIP environments do not require these. + # aiter packs draft-extend support into the decode (multi-step) + # backend; DSV4 exposes it on the draft-extend backend itself. + from sglang.srt.layers.attention.aiter_backend import ( + AiterMultiStepDraftBackend, + ) + from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( + DeepseekV4HipRadixBackend, + ) + from sglang.srt.layers.attention.dsa_backend import ( + DeepseekSparseAttnBackend, + ) + + supports_hip_draft_extend_graph = ( + isinstance(self.draft_attn_backend, AiterMultiStepDraftBackend) + or isinstance(self.draft_extend_attn_backend, DeepseekV4HipRadixBackend) + or isinstance(self.draft_extend_attn_backend, DeepseekSparseAttnBackend) + ) + + graph_supported_backend_types = [ + TritonAttnBackend, + TRTLLMMLABackend, + TRTLLMHAAttnBackend, + TokenspeedMLABackend, + FlashInferAttnBackend, + ] + if _is_cuda or _is_musa: + # DSA is CUDA-only; import lazily so non-CUDA builds don't pull in + # deep_gemm and the rest of the sparse-attention stack at import time. + from sglang.srt.layers.attention.dsa_backend import ( + DeepseekSparseAttnBackend, + ) + + graph_supported_backend_types.append(DeepseekSparseAttnBackend) + from sglang.srt.layers.attention.deepseek_v4_backend import ( + DeepseekV4AttnBackend, + ) + + graph_supported_backend_types.append(DeepseekV4AttnBackend) + if _is_cuda: + # FlashMLA is CUDA-only; import lazily so CPU builds don't pull + # sgl_kernel.flash_mla at import time. + from sglang.srt.layers.attention.flashmla_backend import FlashMLABackend + + graph_supported_backend_types.append(FlashMLABackend) + + graph_supported_backend = isinstance( + self.draft_extend_attn_backend, + tuple(graph_supported_backend_types), + ) + supports_cuda_draft_extend_graph = ( + _is_cuda or _is_musa + ) and graph_supported_backend + # Capture extend + # TODO: support draft extend cuda graph for more attention backends + if ( + self.draft_extend_attn_backend + and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get() + and ( + _is_npu + or _is_xpu + or supports_cuda_draft_extend_graph + or supports_hip_draft_extend_graph + ) + ): + tic = time.perf_counter() + before_mem = get_available_gpu_memory(self.device, self.gpu_id) + log_info_on_rank0( + logger, + f"Capture draft extend CUDA graph begin. backend={decode_backend}, " + f"num_tokens_per_req={self.speculative_num_draft_tokens}, " + f"bs={capture_bs}, avail mem={before_mem:.2f} GB", + ) + self.cuda_graph_runner_for_draft_extend = Device2ExtendCudaGraphRunner[ + self.target_worker.device + ](self) + # draft_extend is the step's last shared-buffer-reading phase; its + # read-done event is what the scheduler's WAR barrier waits on. + after_mem = get_available_gpu_memory(self.device, self.gpu_id) + capture_time = time.perf_counter() - tic + self._specialized_graph_memory_usage["draft_extend"] = ( + self._specialized_graph_memory_usage.get("draft_extend", 0.0) + + before_mem + - after_mem + ) + self._specialized_graph_time_usage["draft_extend"] = ( + self._specialized_graph_time_usage.get("draft_extend", 0.0) + + capture_time + ) + log_info_on_rank0( + logger, + "Capture draft extend CUDA graph end. " + f"elapsed={capture_time:.2f} s, " + f"mem usage={(before_mem - after_mem):.2f} GB, " + f"avail mem={after_mem:.2f} GB.", + ) + + def draft(self, batch: ScheduleBatch): + draft_input: EagleDraftInput = batch.spec_info + forward_batch, can_run_decode_cuda_graph = prepare_for_draft( + draft_input, + self.req_to_token_pool, + batch, + self.cuda_graph_runner, + self.draft_runner, + self.topk, + self.speculative_num_steps, + ) + if ( + can_run_decode_cuda_graph + and not forward_batch.forward_mode.is_idle() + and self.seed_dsa_topk_from_draft_extend + and draft_input.dsa_topk_indices is None + ): + can_run_decode_cuda_graph = False + # [PPSPEC] debugging aid: force the eager draft-step path to isolate + # draft-decode cuda-graph capture issues on PP stages. + if os.getenv("SGLANG_PP_SPEC_FORCE_EAGER_DRAFT", "0") == "1": + can_run_decode_cuda_graph = False + + n_inner = self.speculative_num_steps - 1 + canary_outside_ctx = ( + c.with_ops_outside_graph( + single_forward_indices=list(range(n_inner)), + maybe_inaccurate_forward_batch=forward_batch, + ) + if (c := self.draft_runner.canary_manager) is not None + else contextlib.nullcontext() + ) + + with canary_outside_ctx: + # Run draft + if can_run_decode_cuda_graph: + parent_list, top_scores_index, draft_tokens, draft_probs = ( + self.cuda_graph_runner.execute(forward_batch) + ) + else: + if ( + not forward_batch.forward_mode.is_idle() + and self.speculative_num_steps > 1 + ): + # Skip attention backend init for 1-step draft, + # `draft_forward` only does sample in this case. + self.draft_attn_backend.init_forward_metadata(forward_batch) + forward_batch.mark_forward_metadata_ready() + parent_list, top_scores_index, draft_tokens, draft_probs = ( + self.draft_forward(forward_batch) + ) + + # [PP+spec] Stage-boundary host sync (unconditional): the debug + # probes' tensor reads below established this ordering; without it + # the eager spec chain races under concurrent load (async CUDA + # illegal access, 8x16384 repro; CUDA_LAUNCH_BLOCKING and debug + # builds both pass, no-drain builds crash). + if _pp_sync_mask() & 0x01: # site1 draftstep + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG: + logger.info( + f"[PPSPEC] draftstep pp_rank={get_pp_group().rank_in_group} " + f"graph={can_run_decode_cuda_graph} " + f"tokens={draft_tokens.view(-1)[:8].tolist()} " + f"pos={forward_batch.positions.view(-1)[:4].tolist()} " + f"in_hidden={None if draft_input.hidden_states is None else f'{draft_input.hidden_states.abs().max().item():.3e}'}" + ) + + return build_eagle_verify_input( + batch, + draft_input, + parent_list, + top_scores_index, + draft_tokens, + draft_probs, + target_worker=self.target_worker, + topk=self.topk, + num_steps=self.speculative_num_steps, + num_draft_tokens=self.speculative_num_draft_tokens, + tree_mask_mode=self.tree_mask_mode, + device=self.device, + ) + + def draft_forward(self, forward_batch: ForwardBatch): + # Parse args + spec_info: EagleDraftInput = forward_batch.spec_info + if forward_batch.forward_mode.is_idle(): + return self._draft_forward_idle(forward_batch, spec_info) + + out_cache_loc = forward_batch.out_cache_loc + topk_p, topk_index, hidden_states = ( + spec_info.topk_p, + spec_info.topk_index, + spec_info.hidden_states, + ) + + maybe_detect_nan(topk_p, "draft_forward: NaN in initial topk_p from spec_info") + + if self.hot_token_id is not None: + topk_index = self.hot_token_id[topk_index] + + out_cache_loc = per_step_draft_out_cache_loc( + out_cache_loc, + forward_batch.batch_size, + self.topk, + self.speculative_num_steps, + ) + + # Return values + score_list: List[torch.Tensor] = [] + token_list: List[torch.Tensor] = [] + parents_list: List[torch.Tensor] = [] + if get_spec().speculative_use_rejection_sampling: + draft_probs_list: List[torch.Tensor] = [spec_info.draft_probs] + + topk1_chain_fits = ( + self.topk == 1 + and topk_index.shape[0] <= self._topk1_parents_prealloc.shape[0] + ) + # Materialize the chain directly only when the CUDA kernel can write + # every subsequent column. Other topk=1 paths retain the token list and + # assemble it with one final cat instead of launching a copy per step. + draft_tokens_topk1 = None + if ( + topk1_chain_fits + and _is_cuda + and self.hot_token_id is None + and not get_spec().speculative_use_rejection_sampling + ): + draft_tokens_topk1 = torch.empty( + (topk_index.shape[0], self.speculative_num_steps), + dtype=topk_index.dtype, + device=topk_index.device, + ) + draft_tokens_topk1[:, :1].copy_(topk_index) + + # Forward multiple steps + scores = None + with IndexTopKShareState.mtp_iteration( + forward_batch, + enabled=self.index_share_for_mtp_iteration, + keep_carry_seed=self.seed_dsa_topk_from_draft_extend, + ): + for i in range(self.speculative_num_steps): + if draft_tokens_topk1 is not None: + input_ids = topk_index.flatten() + else: + input_ids, hidden_states, scores, tree_info = select_top_k_tokens( + i, topk_p, topk_index, hidden_states, scores, self.topk + ) + score_list.append(tree_info[0]) + token_list.append(tree_info[1]) + parents_list.append(tree_info[2]) + + if i == self.speculative_num_steps - 1: + break + + forward_batch.input_ids = input_ids + # Qwen3-MoE MTP uses a fused RoPE + KV-store path whose cache_loc + # argument must be contiguous. + if ( + self.draft_runner.model_config.hf_config.architectures[0] + == "Qwen3MoeForCausalLMMTP" + ): + out_cache_loc = out_cache_loc.contiguous() + forward_batch.out_cache_loc = out_cache_loc[i] + spec_info.hidden_states = hidden_states + + canary_index_ctx = ( + c.with_active_single_forward_manager(i) + if (c := self.draft_runner.canary_manager) is not None + else contextlib.nullcontext() + ) + with ( + forward_context( + ForwardContext( + attn_backend=self.draft_attn_backend.attn_backends[i] + ) + ), + canary_index_ctx, + ): + logits_output = self.draft_runner.forward( + forward_batch + ).logits_output + maybe_detect_nan( + logits_output.next_token_logits, f"draft_forward step {i}" + ) + maybe_detect_inf( + logits_output.next_token_logits, f"draft_forward step {i}" + ) + if get_spec().speculative_use_rejection_sampling: + probs, topk_p, topk_index = sample_draft_proposal( + logits_output.next_token_logits, + forward_batch.sampling_info.temperatures, + ) + draft_probs_list.append(probs) + forward_batch.positions.add_(1) + elif self.topk == 1 and not _is_hip: + if _is_cuda: + topk_p, topk_index = draft_topk1_postprocess( + logits_output.next_token_logits, + forward_batch.positions, + draft_tokens_topk1, + i + 1, + ) + else: + topk_index = torch.argmax( + logits_output.next_token_logits, dim=-1, keepdim=True + ) + topk_p = torch.ones_like(topk_index, dtype=torch.float32) + forward_batch.positions.add_(1) + else: + probs = renorm_draft_probs( + logits_output.next_token_logits, + forward_batch.sampling_info, + get_spec().speculative_use_rejection_sampling, + ) + topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) + forward_batch.positions.add_(1) + maybe_detect_oob( + topk_index, + 0, + logits_output.next_token_logits.shape[-1], + f"draft_forward step {i}: topk_index OOB vs vocab_size={logits_output.next_token_logits.shape[-1]}", + ) + if self.hot_token_id is not None: + topk_index = self.hot_token_id[topk_index] + hidden_states = logits_output.hidden_states + + draft_probs = ( + torch.stack(draft_probs_list, dim=1) + if get_spec().speculative_use_rejection_sampling + else None + ) + + # Organize the results + if draft_tokens_topk1 is not None: + bs = draft_tokens_topk1.shape[0] + top_scores_index = self._topk1_score_indices_prealloc[:bs] + parent_list = self._topk1_parents_prealloc[:bs] + return parent_list, top_scores_index, draft_tokens_topk1, draft_probs + + if topk1_chain_fits: + bs = token_list[0].shape[0] + draft_tokens = torch.cat(token_list, dim=1) + top_scores_index = self._topk1_score_indices_prealloc[:bs] + parent_list = self._topk1_parents_prealloc[:bs] + return parent_list, top_scores_index, draft_tokens, draft_probs + + parent_list, top_scores_index, draft_tokens = organize_draft_results( + score_list, token_list, parents_list, self.speculative_num_draft_tokens + ) + + return parent_list, top_scores_index, draft_tokens, draft_probs + + def _draft_forward_idle( + self, forward_batch: ForwardBatch, spec_info: EagleDraftInput + ): + """Run eager idle-rank collectives without materializing draft state.""" + input_ids = forward_batch.input_ids + out_cache_loc = forward_batch.out_cache_loc + hidden_states = spec_info.hidden_states + + # ModelRunner pads and unpads the empty batch on every call. Avoid the + # normal tree/cache-layout path: idle outputs are discarded when the + # verify input is built, but every rank must still enter each forward. + for i in range(self.speculative_num_steps - 1): + forward_batch.input_ids = input_ids + forward_batch.out_cache_loc = out_cache_loc + spec_info.hidden_states = hidden_states + canary_index_ctx = ( + c.with_active_single_forward_manager(i) + if (c := self.draft_runner.canary_manager) is not None + else contextlib.nullcontext() + ) + with ( + forward_context( + ForwardContext( + attn_backend=self.draft_attn_backend.attn_backends[i] + ) + ), + canary_index_ctx, + ): + self.draft_runner.forward(forward_batch) + + return None, None, None, None + + def draft_extend(self): + pass + + def _draft_extend_for_prefill( + self, + batch: ScheduleBatch, + target_hidden_states: torch.Tensor, + next_token_ids: torch.Tensor, + mm_input_embeds: Optional[torch.Tensor] = None, + prebuilt_input_ids: Optional[torch.Tensor] = None, + ): + """ + Run draft model extend to correctly fill the KV cache. + + Args: + batch: The batch to run. + target_hidden_states: Hidden states from the target model forward + next_token_ids: Next token ids generated from the target forward. + prebuilt_input_ids: [EXPERIMENT] PP+spec replay — the last stage's + already-shifted draft-extend input ids, relayed via the output + ring. The non-last stage's own batch.input_ids was cleared by + run_batch before the ring pack, so it cannot re-derive them. + """ + # Construct input_ids + if not batch.forward_mode.is_idle(): + if prebuilt_input_ids is not None: + batch.input_ids = prebuilt_input_ids + else: + # Chunked-prefill-aware tail tokens (see PR #26329). + tail_tokens = _eagle_prefill_tail_tokens(batch, next_token_ids) + new_input_ids = torch.empty_like(batch.input_ids) + pt = 0 + for i, extend_len in enumerate(batch.extend_lens): + input_ids = batch.input_ids[pt : pt + extend_len] + new_input_ids[pt : pt + extend_len].copy_( + torch.cat((input_ids[1:], tail_tokens[i].reshape(1))) + ) + pt += extend_len + assert pt == batch.input_ids.numel() + batch.input_ids = new_input_ids + + # Draft-extend spec_info for the extend forward; carries only + # hidden_states + shape info. + batch.spec_info = EagleDraftExtendInput( + hidden_states=target_hidden_states, + # draft mode is same with decode mode, only 1 token per req + num_tokens_per_req=1, + num_tokens_for_logprob_per_req=1, + ) + + # Run forward (LAST mode: only the final hidden state per request, + # to feed the next draft step which expects [bs, hidden_dim]). + # STANDALONE skips hidden states end-to-end. + capture_hidden_mode = ( + CaptureHiddenMode.NULL + if self.speculative_algorithm.is_standalone() + else CaptureHiddenMode.LAST + ) + forward_batch = ForwardBatch.init_new( + batch, + self.draft_runner, + capture_hidden_mode=capture_hidden_mode, + return_hidden_states_before_norm=False, + ) + forward_batch.return_logprob = False + if mm_input_embeds is not None: + forward_batch.mm_input_embeds = mm_input_embeds + + # Seed the first draft-decode loop from each request's last prefill + # position. Gather last-per-req before the copy (prefill can be long). + seed_from_extend = ( + self.seed_dsa_topk_from_draft_extend + and not forward_batch.forward_mode.is_idle() + ) + if seed_from_extend: + bs = forward_batch.batch_size + forward_batch.spec_info.dsa_seed_topk_capture = ( + self._get_dsa_extend_topk_buf(bs) + ) + forward_batch.spec_info.dsa_seed_topk_select = ( + torch.cumsum(forward_batch.extend_seq_lens, dim=0) - 1 + ).long() + + canary_ctx = ( + context_tuple( + c.with_ops_outside_graph( + single_forward_indices=[0], + maybe_inaccurate_forward_batch=forward_batch, + ), + c.with_active_single_forward_manager(0), + ) + if (c := self.draft_runner.canary_manager) is not None + else contextlib.nullcontext() + ) + with canary_ctx: + logits_output = self.draft_runner.forward(forward_batch).logits_output + maybe_detect_nan(logits_output.next_token_logits, "draft_extend_for_prefill") + maybe_detect_inf(logits_output.next_token_logits, "draft_extend_for_prefill") + + prefill_dsa_topk = None + if seed_from_extend: + prefill_dsa_topk = self.dsa_extend_topk_buf[:bs].clone() + + # Assemble the next-iter draft spec_info from the extend output. + use_rejection_sampling = get_spec().speculative_use_rejection_sampling + probs = renorm_draft_probs( + logits_output.next_token_logits, + batch.sampling_info, + use_rejection_sampling, + ) + if use_rejection_sampling: + topk_p, topk_index = fast_sample(probs, num_samples=1) + else: + topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) + # [PP+spec] Stage-boundary host sync (unconditional) — see the + # draftstep note for the rationale. + if _pp_sync_mask() & 0x02: # site2 prefill-ext + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG: + _hs = logits_output.hidden_states + _vh = None if _hs is None else f"{_hs.abs().max().item():.3e}" + logger.info( + f"[PPSPEC] prefill-ext pp_rank={get_pp_group().rank_in_group} " + f"dl_max={logits_output.next_token_logits.abs().max().item():.3e} " + f"topk={topk_index.view(-1)[:4].tolist()} " + f"vh_max={_vh}" + ) + if not getattr(self, "_pp_lm_head_probed", False): + self._pp_lm_head_probed = True + _lh = getattr(self.draft_runner.model, "lm_head", None) + _tw = getattr(self, "target_worker", None) + _tlh = ( + _tw.model_runner.model.lm_head + if _tw is not None + and getattr(_tw.model_runner, "model", None) is not None + and hasattr(_tw.model_runner.model, "lm_head") + else None + ) + _lh_w = None if _lh is None else _lh.weight + _tlh_w = ( + None + if _tlh is None or not hasattr(_tlh, "weight") + else _tlh.weight + ) + logger.info( + f"[PPSPEC] lmhead pp_rank={get_pp_group().rank_in_group} " + f"draft_shape={None if _lh_w is None else tuple(_lh_w.shape)} " + f"draft_dtype={None if _lh_w is None else _lh_w.dtype} " + f"draft_absmax={None if _lh_w is None else f'{_lh_w.abs().max().item():.3e}'} " + f"target_absmax={None if _tlh_w is None else f'{_tlh_w.abs().max().item():.3e}'}" + ) + return EagleDraftInput( + topk_p=topk_p, + topk_index=topk_index, + draft_probs=probs if use_rejection_sampling else None, + hidden_states=logits_output.hidden_states, + bonus_tokens=next_token_ids, + num_tokens_per_req=1, + num_tokens_for_logprob_per_req=1, + dsa_topk_indices=prefill_dsa_topk, + ) + + def _get_dsa_extend_topk_buf(self, num_tokens: int) -> torch.Tensor: + """Lazily-grown int32 [num_tokens, index_topk] eager draft-extend seed buffer.""" + buf = self.dsa_extend_topk_buf + if buf is None or buf.shape[0] < num_tokens: + buf = torch.full( + (num_tokens, self.dsa_index_topk), + -1, + dtype=torch.int32, + device=self.device, + ) + self.dsa_extend_topk_buf = buf + return buf[:num_tokens] + + def _draft_extend_for_decode( + self, batch: ScheduleBatch, batch_result: GenerationBatchResult + ): + # Batch 2: Draft extend + draft_extend_input = EagleDraftExtendInput( + hidden_states=batch_result.logits_output.hidden_states, + # accept_lens includes the bonus token; correct drafts exclude it. + num_correct_drafts=batch_result.accept_lens - 1, + num_accept_tokens=batch_result.accept_lens, + # Draft-extend fills the whole tree width (num_draft_tokens) per req, + # not num_steps + 1, so DP MLP-sync padding stays consistent for topk > 1. + num_tokens_per_req=self.speculative_num_draft_tokens, + num_tokens_for_logprob_per_req=self.speculative_num_draft_tokens, + ) + select_index = ( + torch.arange( + 0, + len(batch.seq_lens) * self.speculative_num_draft_tokens, + self.speculative_num_draft_tokens, + device=self.device, + ) + + batch_result.accept_lens + - 1 + ) + + # Cast to int64 before entering plan stream to avoid cross-stream + # synchronization issues with .to() inside the plan stream context. + next_token_ids = batch_result.next_token_ids.to(torch.int64) + + # Prepare for draft extend in a separate stream + with self.plan_stream_ctx: + forward_batch = prepare_for_draft_extend( + draft_extend_input, + batch, + next_token_ids, + self.speculative_num_draft_tokens, + self.draft_runner, + self.cuda_graph_runner_for_draft_extend, + return_hidden_states_before_norm=False, + ) + + if self.plan_stream: + torch.get_device_module(self.device).current_stream().wait_stream( + self.plan_stream + ) + + # Run draft extend batch in the main compute stream + can_run_decode_cuda_graph = ( + self.cuda_graph_runner_for_draft_extend + and self.cuda_graph_runner_for_draft_extend.can_run_graph(forward_batch) + ) + # [PP+spec] Stage-boundary host sync (unconditional) — see the + # draftstep note for the rationale. + if _pp_sync_mask() & 0x04: # site3 ext-pre + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG: + logger.info( + f"[PPSPEC] ext pp_rank={get_pp_group().rank_in_group} " + f"vh_max={batch_result.logits_output.hidden_states.abs().max().item():.3e} " + f"hidden={tuple(batch_result.logits_output.hidden_states.shape)} " + f"accept={tuple(batch_result.accept_lens.shape)}/{batch_result.accept_lens.tolist()[:8]} " + f"nti={tuple(next_token_ids.shape)} " + f"bs={len(batch.seq_lens)} fb_ids={tuple(forward_batch.input_ids.shape)} " + f"fb_bs={forward_batch.batch_size} fb_seq={tuple(forward_batch.seq_lens.shape)} " + f"fb_pos={tuple(forward_batch.positions.shape)} " + f"fb_ocl={tuple(forward_batch.out_cache_loc.shape)} " + f"fb_ocl_v={forward_batch.out_cache_loc.tolist()[:8]} " + f"sl_v={batch.seq_lens.tolist()[:8]} " + f"fb_rp={tuple(forward_batch.req_pool_indices.shape)} " + f"fb_esl={forward_batch.extend_seq_lens} " + f"ncd={tuple(forward_batch.spec_info.num_correct_drafts.shape)} " + f"nat={tuple(forward_batch.spec_info.num_accept_tokens.shape)} " + f"graph={can_run_decode_cuda_graph}" + ) + + # Eager path publishes the indexer top-k into a worker buffer (the graph + # path uses the runner's static buffer). Gathered at select_index below. + if self.seed_dsa_topk_from_draft_extend and not can_run_decode_cuda_graph: + forward_batch.spec_info.dsa_seed_topk_capture = ( + self._get_dsa_extend_topk_buf(forward_batch.input_ids.shape[0]) + ) + + canary_ctx = ( + context_tuple( + c.with_ops_outside_graph( + single_forward_indices=[0], + maybe_inaccurate_forward_batch=forward_batch, + ), + c.with_active_single_forward_manager(0), + ) + if (c := self.draft_runner.canary_manager) is not None + else contextlib.nullcontext() + ) + with canary_ctx: + if can_run_decode_cuda_graph: + draft_logits_output = self.cuda_graph_runner_for_draft_extend.execute( + forward_batch + ) + else: + draft_logits_output = self.draft_runner.forward( + forward_batch + ).logits_output + + # [PP+spec] Stage-boundary host sync (unconditional) — see the + # draftstep note for the rationale. + if _pp_sync_mask() & 0x08: # site4 extlogits + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG: + logger.info( + f"[PPSPEC] extlogits pp_rank={get_pp_group().rank_in_group} " + f"dl_max={draft_logits_output.next_token_logits.abs().max().item():.3e}" + ) + + maybe_detect_nan( + draft_logits_output.next_token_logits, + f"draft_extend_for_decode (cuda_graph={can_run_decode_cuda_graph})", + ) + maybe_detect_inf( + draft_logits_output.next_token_logits, + f"draft_extend_for_decode (cuda_graph={can_run_decode_cuda_graph})", + ) + + # Gather the per-request last-position indexer top-k as the next loop's + # seed (select_index already picks the last accepted position per req). + dsa_seed_topk_indices = None + if self.seed_dsa_topk_from_draft_extend: + if can_run_decode_cuda_graph: + dsa_extend_topk_capture = ( + self.cuda_graph_runner_for_draft_extend.buffers.dsa_seed_topk_capture + ) + else: + dsa_extend_topk_capture = forward_batch.spec_info.dsa_seed_topk_capture + # Fancy indexing returns a fresh tensor (detached from the buffer). + dsa_seed_topk_indices = dsa_extend_topk_capture[select_index] + + # Reorganize the spec info for the next batch + draft_logits_output.next_token_logits = draft_logits_output.next_token_logits[ + select_index + ] + if draft_logits_output.hidden_states is not None: + draft_logits_output.hidden_states = draft_logits_output.hidden_states[ + select_index + ] + # The draft-extend graph only anchors full logits; selected-row topk is + # owned by the worker for both graph and eager paths. + if get_spec().speculative_use_rejection_sampling: + ret_draft_probs, ret_topk_p, ret_topk_index = sample_draft_proposal( + draft_logits_output.next_token_logits, + batch.sampling_info.temperatures, + ) + elif self.topk == 1 and not _is_hip: + # Gated to CUDA: see #26358 — ROCm's argmax tie-break corrupts + # MTP draft selection on FP8 logits. + ret_topk_index = torch.argmax( + draft_logits_output.next_token_logits, dim=-1, keepdim=True + ) + ret_topk_p = torch.ones_like(ret_topk_index, dtype=torch.float32) + ret_draft_probs = None + else: + probs = renorm_draft_probs( + draft_logits_output.next_token_logits, + batch.sampling_info, + get_spec().speculative_use_rejection_sampling, + ) + ret_topk_p, ret_topk_index = fast_topk(probs, self.topk, dim=-1) + ret_draft_probs = None + ret_hidden_states = draft_logits_output.hidden_states + + # Construct the return values + next_draft_input = batch_result.next_draft_input + ( + next_draft_input.topk_p, + next_draft_input.topk_index, + next_draft_input.hidden_states, + ) = ( + ret_topk_p, + ret_topk_index, + ret_hidden_states, + ) + if get_spec().speculative_use_rejection_sampling: + next_draft_input.draft_probs = ret_draft_probs + if self.seed_dsa_topk_from_draft_extend: + next_draft_input.dsa_topk_indices = dsa_seed_topk_indices + + +class EAGLEWorkerV2(BaseSpecWorker): + def __init__( + self, + server_args: ServerArgs, + gpu_id: int, + ps: ParallelState, + nccl_port: int, + target_worker: TpModelWorker, + ): + super().__init__() + + # Parse arguments + self.server_args = server_args + self.topk = get_spec().speculative_eagle_topk + self.speculative_num_steps = get_spec().speculative_num_steps + self.speculative_num_draft_tokens = get_spec().speculative_num_draft_tokens + self.ps = ps + self.gpu_id = gpu_id + self.device = get_device().device + self._target_worker = target_worker + self.page_size = get_schedule().page_size + self.speculative_algorithm = SpeculativeAlgorithm.from_string( + get_spec().speculative_algorithm + ) + + # [EXPERIMENT] PP+spec replica drafts: EVERY PP stage hosts a full + # draft replica. The last stage still owns sampling/publish; non-last + # stages run identical drafts (deterministic at topk=1: same weights, + # same relayed roots, same TP4 sharding) and maintain their draft KV + # via the scheduler-side replay of the ring-relayed round state + # (scheduler_pp_mixin._pp_prep_batch_result). _hosts_draft below keeps + # the rank-role distinction for publish/accept ownership. + self._hosts_draft = get_pp_group().is_last_rank + self._draft_worker = EagleDraftWorker( + server_args, + gpu_id, + ps, + nccl_port, + target_worker, + ) + + # Adaptive speculative + self.adaptive_controller: Optional[AdaptiveController] = None + if get_spec().speculative_adaptive and self._hosts_draft: + self.adaptive_controller = AdaptiveController( + self, + config_path=get_spec().speculative_adaptive_config, + ) + + # Some dummy tensors + self.num_new_pages_per_topk = torch.empty( + (), dtype=torch.int64, device=self.device + ) + self.extend_lens = torch.empty((), dtype=torch.int64, device=self.device) + + self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) + + @property + def last_shared_read_runner(self): + # Per the base contract: the step's last shared-buffer-reading phase is + # draft_extend, which runs on the draft runner. + return self._draft_worker.draft_runner + + @property + def spec_v2_attn_backends(self) -> tuple: + # Every attn backend a spec_v2 forward touches; consumed by + # decide_needs_cpu_seq_lens to gate the seq_lens_cpu D2H. + return ( + self._target_worker.model_runner.attn_backend, + self._draft_worker.draft_attn_backend, + self._draft_worker.draft_extend_attn_backend + or self._draft_worker.draft_runner.attn_backend, + ) + + def init_cuda_graphs(self): + super().init_cuda_graphs() + # Build adaptive runtime states after target and draft backends exist. + if self.adaptive_controller is not None: + with ( + self._draft_worker.draft_tp_context( + self._draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + ): + self.adaptive_controller.register( + SpecRuntimeState( + speculative_num_steps=self.speculative_num_steps, + speculative_num_draft_tokens=self.speculative_num_draft_tokens, + draft_attn_backend=self._draft_worker.draft_attn_backend, + cuda_graph_runner=self._draft_worker.cuda_graph_runner, + target_attn_backend=self._target_worker.model_runner.attn_backend, + target_graph_runner=self._target_worker.model_runner.decode_cuda_graph_runner, + draft_extend_attn_backend=self._draft_worker.draft_extend_attn_backend, + cuda_graph_runner_for_draft_extend=self._draft_worker.cuda_graph_runner_for_draft_extend, + ) + ) + self.adaptive_controller.init_states( + cuda_graph_bs=( + None + if check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED) + else get_exec().graph.cuda_graph_bs_decode + ), + ) + + def forward_batch_generation( + self, + batch: ScheduleBatch, + on_publish=None, + grammar_barrier=None, + pp_proxy_tensors=None, + ): + if batch.forward_mode.is_extend() or batch.is_extend_in_batch: + # Target prefill + target_capture_mode = ( + CaptureHiddenMode.NULL + if self.speculative_algorithm.is_standalone() + else CaptureHiddenMode.FULL + ) + batch_output = self.target_worker.forward_batch_generation( + batch, + pp_proxy_tensors=pp_proxy_tensors, + capture_hidden_mode=target_capture_mode, + ) + + # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. + # Extend processed L prompt tokens; next verify iter expects same L. + batch_output.new_seq_lens = batch.seq_lens + # Publish before draft_extend so the fence is at target-end. + if on_publish is not None: + on_publish(batch_output.new_seq_lens) + + # [EXPERIMENT] PP+spec: non-last stages skip the in-forward draft + # extend; their draft replica is maintained by the scheduler-side + # replay of the ring-relayed round state instead. + if not self._hosts_draft: + return batch_output + + # Draft prefill + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft_extend"), + ): + batch_output.next_draft_input = ( + self.draft_worker._draft_extend_for_prefill( + batch, + batch_output.logits_output.hidden_states, + batch_output.next_token_ids, + batch_output.logits_output.mm_input_embeds, + ) + ) + # [EXPERIMENT] PP+spec: stash the shifted draft-extend input + # ids for the output ring — the non-last stages' replay needs + # them verbatim (their batch.input_ids is cleared by run_batch + # before the ring pack, and the packer runs after run_batch). + batch_output.next_draft_input.spec_prefill_input_ids = ( + batch.input_ids + ) + return batch_output + else: + self.activate_step_by_batch(batch.seq_lens.shape[0]) + + if batch.spec_info is None: + capture_mode = ( + CaptureHiddenMode.NULL + if self.speculative_algorithm.is_standalone() + else CaptureHiddenMode.LAST + ) + hidden_size, hidden_dtype = get_draft_recurrent_hidden_state_spec( + self.draft_worker.draft_runner + ) + batch.spec_info = EagleDraftInput.create_idle_input( + device=self.device, + hidden_size=hidden_size, + dtype=hidden_dtype, + topk=self.topk, + capture_hidden_mode=capture_mode, + vocab_size=self.target_worker.model_config.vocab_size, + ) + if self.speculative_num_steps == 0: + # Drafting disabled (high batch size). _draft_extend below still + # runs, keeping draft KV warm for when the batch shrinks. + verify_input = self._build_trivial_verify_input(batch) + else: + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft"), + ): + verify_input: EagleVerifyInput = self.draft_worker.draft(batch) + assert verify_input.is_verify_input() + # [PP+spec] Stage-boundary host sync (unconditional) — see the + # draftstep note for the rationale. + if _pp_sync_mask() & 0x10: # site5 draft-tree + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG: + logger.info( + f"[PPSPEC] pp_rank={get_pp_group().rank_in_group} " + f"bs={batch.seq_lens.shape[0]} " + f"tree_sum={int(verify_input.draft_token.sum().item())} " + f"tree_head={verify_input.draft_token[: min(8, verify_input.draft_token.numel())].tolist()}" + ) + batch.spec_info = verify_input + batch_output = self.verify( + batch, + grammar_barrier=grammar_barrier, + pp_proxy_tensors=pp_proxy_tensors, + ) + # [EXPERIMENT] PP+spec: capture the verify-assigned draft-KV slots + # (batch.out_cache_loc = the tree rows) onto the result NOW — + # run_batch's _forward_isolation reverts the in-forward batch + # edits afterwards, so the scheduler-side replay on non-last + # stages can no longer read them off the batch. Relayed via the + # output ring as "spec_verify_slots". + batch_output.pp_verify_slots = batch.out_cache_loc + # Publish before draft_extend so the fence is at verify-end. + # [EXPERIMENT] PP+spec: non-last stages return proxies-only from + # verify (no new_seq_lens); only the draft-hosting last stage + # publishes. + if on_publish is not None and self._hosts_draft: + on_publish(batch_output.new_seq_lens) + if ( + self.speculative_num_steps == 0 + and envs.SGLANG_SPEC_SKIP_ZERO_STEP_DRAFT_EXTEND.get() + ): + if self._hosts_draft: + self._stub_skipped_draft_extend(batch, batch_output) + elif self._hosts_draft: + # [EXPERIMENT] PP+spec: non-last stages skip the in-forward + # draft-extend; the scheduler replays it from the ring-relayed + # round state (scheduler_pp_mixin._pp_prep_batch_result). + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft_extend"), + ): + self.draft_worker._draft_extend_for_decode(batch, batch_output) + # [PP+spec] Stage-boundary host sync (unconditional, gated the + # same way as the debug reads) — see the draftstep note. + if self._hosts_draft and (_pp_sync_mask() & 0x20): # site6 accept + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG and self._hosts_draft: + logger.info( + f"[PPSPEC] pp_rank={get_pp_group().rank_in_group} " + f"accept_mean={batch_output.accept_lens.float().mean().item():.3f} " + f"accept_head={batch_output.accept_lens.tolist()[:16]}" + ) + + return batch_output + + def _build_trivial_verify_input(self, batch: ScheduleBatch) -> EagleVerifyInput: + """Build a 1-node EagleVerifyInput rooted at the previous bonus token. + + Used when ``speculative_num_steps == 0`` to skip drafting while still + routing through the existing TARGET_VERIFY graph captured at + ``draft_token_num=1``: the kernel always accepts the root and samples + one new bonus token from target logits -- functionally a plain decode. + """ + if batch.forward_mode.is_idle(): + return EagleVerifyInput.create_idle_input( + topk=self.topk, spec_steps=0, num_verify_tokens=1, device=self.device + ) + + draft_input: EagleDraftInput = batch.spec_info + bs = batch.seq_lens.shape[0] + device = self.device + + retrieve_index = torch.arange(bs, dtype=torch.long, device=device).unsqueeze(1) + retrieve_next_token = torch.full((bs, 1), -1, dtype=torch.long, device=device) + retrieve_next_sibling = torch.full((bs, 1), -1, dtype=torch.long, device=device) + + attn_backend = self._target_worker.model_runner.attn_backend + verify_mask = attn_backend.verify_mask + # Every position in a 1-node tree is visible, so an all-True fill is + # correct under either layout. + if verify_mask is not None and verify_mask.fits(bs): + custom_mask = verify_mask.buffer + custom_mask.fill_(True) + else: + if batch.seq_lens_sum is not None: + seq_lens_sum = batch.seq_lens_sum + elif batch.seq_lens_cpu is not None: + seq_lens_sum = int(batch.seq_lens_cpu.sum()) + else: + seq_lens_sum = bs * attn_backend.max_context_len + custom_mask = torch.ones(seq_lens_sum + bs, dtype=torch.bool, device=device) + + positions = batch.seq_lens.to(torch.int64) + + return EagleVerifyInput( + draft_token=draft_input.bonus_tokens, + custom_mask=custom_mask, + positions=positions, + retrieve_index=retrieve_index, + retrieve_next_token=retrieve_next_token, + retrieve_next_sibling=retrieve_next_sibling, + retrieve_cum_len=None, + spec_steps=0, + topk=self.topk, + draft_token_num=1, + capture_hidden_mode=CaptureHiddenMode.FULL, + seq_lens_sum=None, + seq_lens_cpu=None, + ) + + def _stub_skipped_draft_extend( + self, batch: ScheduleBatch, batch_output: GenerationBatchResult + ) -> None: + """Fill shape-valid stubs on next_draft_input when draft_extend is skipped. + + ``verify`` already set ``bonus_tokens`` (the only field the next steps=0 + verify reads). The overlap FutureMap still stashes topk_p/topk_index/ + hidden_states, so provide zeroed tensors of the right shape. They are never + consumed while at steps=0; an upshift to steps>0 would draft from this stale + state (cold recovery), which is the documented cost of this experimental flag. + """ + next_draft_input: EagleDraftInput = batch_output.next_draft_input + bs = batch.seq_lens.shape[0] + device = self.device + next_draft_input.topk_p = torch.zeros( + (bs, self.topk), dtype=torch.float32, device=device + ) + next_draft_input.topk_index = torch.zeros( + (bs, self.topk), dtype=torch.int64, device=device + ) + hidden_size, hidden_dtype = get_draft_recurrent_hidden_state_spec( + self.draft_worker.draft_runner + ) + if hidden_size is not None: + next_draft_input.hidden_states = torch.zeros( + (bs, hidden_size), + dtype=hidden_dtype, + device=device, + ) + + def on_verify_complete_cpu( + self, num_correct_drafts_per_req: list[int], batch_size: int = 0 + ) -> None: + if self.adaptive_controller is not None: + self.adaptive_controller.on_verify_complete( + num_correct_drafts_per_req, batch_size=batch_size + ) + + def activate_step_by_batch(self, batch_size: int) -> None: + if self.adaptive_controller is not None: + self.adaptive_controller.activate_step_by_batch(batch_size) + + # -- Adaptive speculative decoding protocol -- + + def build_adaptive_runtime_state( + self, + speculative_num_steps: int, + speculative_num_draft_tokens: int, + cuda_graph_bs=None, + ) -> SpecRuntimeState: + """Build a SpecRuntimeState for the given step configuration.""" + tic = time.perf_counter() + before_mem = get_available_gpu_memory(self.device, self.gpu_id) + + with self._override_worker_state( + speculative_num_steps, + speculative_num_draft_tokens, + cuda_graph_bs=cuda_graph_bs, + ): + self._draft_worker.init_attention_backend() + self._draft_worker._capture_cuda_graphs() + + # Build target attention backend and CUDA graph runner + target_model_runner = self._target_worker.model_runner + backup_init = target_model_runner.init_new_workspace + try: + target_attn_backend = target_model_runner._get_attention_backend( + init_new_workspace=True + ) + finally: + target_model_runner.init_new_workspace = backup_init + + target_graph_runner = None + if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED): + TargetGraphRunnerCls = ( + NPUGraphRunner if _is_npu else DecodeCudaGraphRunner + ) + target_graph_before_mem = get_available_gpu_memory( + self.device, self.gpu_id + ) + target_graph_tic = time.perf_counter() + target_graph_runner = TargetGraphRunnerCls( + target_model_runner, + attn_backend=target_attn_backend, + speculative_num_steps=speculative_num_steps, + speculative_num_draft_tokens=speculative_num_draft_tokens, + ) + target_graph_after_mem = get_available_gpu_memory( + self.device, self.gpu_id + ) + target_graph_time = time.perf_counter() - target_graph_tic + self._additional_graph_memory_usage["target_verify"] = ( + self._additional_graph_memory_usage.get("target_verify", 0.0) + + target_graph_before_mem + - target_graph_after_mem + ) + self._additional_graph_time_usage["target_verify"] = ( + self._additional_graph_time_usage.get("target_verify", 0.0) + + target_graph_time + ) + + state = SpecRuntimeState( + speculative_num_steps=speculative_num_steps, + speculative_num_draft_tokens=speculative_num_draft_tokens, + draft_attn_backend=self._draft_worker.draft_attn_backend, + cuda_graph_runner=self._draft_worker.cuda_graph_runner, + target_attn_backend=target_attn_backend, + target_graph_runner=target_graph_runner, + draft_extend_attn_backend=self._draft_worker.draft_extend_attn_backend, + cuda_graph_runner_for_draft_extend=self._draft_worker.cuda_graph_runner_for_draft_extend, + ) + + after_mem = get_available_gpu_memory(self.device, self.gpu_id) + log_info_on_rank0( + logger, + f"Built adaptive runtime state steps={speculative_num_steps}: " + f"elapsed={time.perf_counter() - tic:.2f}s, " + f"mem={(before_mem - after_mem):.2f}GB", + ) + + return state + + def apply_runtime_state(self, state: SpecRuntimeState) -> None: + """Apply a pre-built runtime state to this worker.""" + if self.speculative_num_steps == state.speculative_num_steps: + return + + log_info_on_rank0( + logger, + "Switch adaptive runtime state: " + f"steps {self.speculative_num_steps} -> {state.speculative_num_steps}, " + f"draft_tokens {self.speculative_num_draft_tokens} -> " + f"{state.speculative_num_draft_tokens}", + ) + + # Top-level + self.speculative_num_steps = state.speculative_num_steps + self.speculative_num_draft_tokens = state.speculative_num_draft_tokens + + # Draft side + dw = self._draft_worker + dw.speculative_num_steps = state.speculative_num_steps + dw.speculative_num_draft_tokens = state.speculative_num_draft_tokens + dw.draft_attn_backend = state.draft_attn_backend + dw.draft_runner.draft_attn_backend = state.draft_attn_backend + dw.cuda_graph_runner = state.cuda_graph_runner + dw.draft_extend_attn_backend = state.draft_extend_attn_backend + # Keep the runner's attn_backend in step with the active draft-extend + # backend (the draft-extend forward reads draft_runner.attn_backend); + # mirrors init_attention_backend. When None, the runner keeps its + # initialized backend (consistent across step configs). + if state.draft_extend_attn_backend is not None: + dw.draft_runner.attn_backend = state.draft_extend_attn_backend + dw.cuda_graph_runner_for_draft_extend = state.cuda_graph_runner_for_draft_extend + dw._rebuild_topk1_chain_buffers() + + # Target side + self._target_worker.model_runner.attn_backend = state.target_attn_backend + self._target_worker.model_runner.decode_cuda_graph_runner = ( + state.target_graph_runner + ) + + # Sync server_args + get_context().override( + "adaptive_spec.restore", + speculative_num_steps=state.speculative_num_steps, + speculative_num_draft_tokens=state.speculative_num_draft_tokens, + ) + + @contextlib.contextmanager + def _override_worker_state( + self, + speculative_num_steps: int, + speculative_num_draft_tokens: int, + cuda_graph_bs: list[int] | None = None, + ): + """Temporarily override server_args and worker attributes for graph capture.""" + dw = self._draft_worker + backup = ( + self.speculative_num_steps, + self.speculative_num_draft_tokens, + dw.speculative_num_steps, + dw.speculative_num_draft_tokens, + dw.draft_attn_backend, + dw.draft_extend_attn_backend, + dw.draft_runner.draft_attn_backend, + dw.draft_runner.attn_backend, + dw.cuda_graph_runner, + dw.cuda_graph_runner_for_draft_extend, + get_spec().speculative_num_steps, + get_spec().speculative_num_draft_tokens, + get_exec().graph.cuda_graph_bs_decode, + get_exec().graph.disable_cuda_graph, + ) + + self.speculative_num_steps = speculative_num_steps + self.speculative_num_draft_tokens = speculative_num_draft_tokens + dw.speculative_num_steps = speculative_num_steps + dw.speculative_num_draft_tokens = speculative_num_draft_tokens + get_context().override( + "adaptive_spec.capture_override", + speculative_num_steps=speculative_num_steps, + speculative_num_draft_tokens=speculative_num_draft_tokens, + ) + if cuda_graph_bs is not None: + # BS-aware adaptive spec may prune cuda_graph_bs to an empty list + # for steps that no BS range uses (e.g. step=1). Disable graph + # capture for those steps; restore in finally so subsequent steps + # are not affected. + get_context().override( + "adaptive_spec.capture_override", + cuda_graph_bs_decode=cuda_graph_bs, + **({"disable_cuda_graph": True} if not cuda_graph_bs else {}), + ) + dw._rebuild_topk1_chain_buffers() + + try: + yield + finally: + ( + self.speculative_num_steps, + self.speculative_num_draft_tokens, + dw.speculative_num_steps, + dw.speculative_num_draft_tokens, + dw.draft_attn_backend, + dw.draft_extend_attn_backend, + dw.draft_runner.draft_attn_backend, + dw.draft_runner.attn_backend, + dw.cuda_graph_runner, + dw.cuda_graph_runner_for_draft_extend, + ) = backup[:10] + get_context().override( + "adaptive_spec.capture_restore", + speculative_num_steps=backup[10], + speculative_num_draft_tokens=backup[11], + cuda_graph_bs_decode=backup[12], + disable_cuda_graph=backup[13], + ) + dw._rebuild_topk1_chain_buffers() + + def verify( + self, + batch: ScheduleBatch, + grammar_barrier=None, + pp_proxy_tensors=None, + ): + return run_eagle_verify( + batch, + target_worker=self.target_worker, + req_to_token_pool=self.req_to_token_pool, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + plan_stream=self.plan_stream, + plan_stream_ctx=self.plan_stream_ctx, + topk=self.topk, + num_draft_tokens=self.speculative_num_draft_tokens, + device=self.device, + metadata_ready_pre_pad=False, + finalize_tree_path=True, + grammar_barrier=grammar_barrier, + pp_proxy_tensors=pp_proxy_tensors, + ) + + # ------------------------------------------------------------------ + # [EXPERIMENT] PP+spec replica maintenance (non-last stages). + # + # The last stage runs draft-extend in-forward (forward_batch_generation). + # Non-last stages cannot: their verify result is proxies-only, so the + # scheduler calls the replay methods below from _pp_prep_batch_result, + # right after the ring rebind and on the same pre-filter batch state the + # last stage saw in-forward (before process_batch_result filters/frees + # anything). This keeps every stage's draft KV + DSA seeds in lockstep, + # so all draft replicas produce identical trees from identical roots. + # + # Both replays snapshot/restore the full set of ScheduleBatch fields the + # draft-extend path mutates, mirroring how _forward_isolation reverts the + # worker's in-forward edits on the last stage before the scheduler's + # explicit carries (spec_info rebind / input_ids=None) are applied. + # ------------------------------------------------------------------ + + _PP_REPLAY_RESTORE_FIELDS = ( + "spec_info", + "input_ids", + "forward_mode", + "prefix_lens", + "extend_lens", + "extend_num_tokens", + "out_cache_loc", + ) + + def replay_draft_extend_for_prefill( + self, + batch: ScheduleBatch, + target_hidden_states: torch.Tensor, + next_token_ids: torch.Tensor, + rebound_spec_info, + prebuilt_input_ids=None, + ): + saved = {f: getattr(batch, f, None) for f in self._PP_REPLAY_RESTORE_FIELDS} + try: + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + ): + ret = self.draft_worker._draft_extend_for_prefill( + batch, + target_hidden_states, + next_token_ids, + prebuilt_input_ids=prebuilt_input_ids, + ) + finally: + for f, v in saved.items(): + setattr(batch, f, v) + # Mirror the last stage's returned dsa_topk_indices onto the rebound + # spec_info (the standard ring rebind drops this field). + if ( + rebound_spec_info is not None + and getattr(ret, "dsa_topk_indices", None) is not None + ): + rebound_spec_info.dsa_topk_indices = ret.dsa_topk_indices + + def replay_draft_extend_for_decode( + self, + batch: ScheduleBatch, + verify_hidden_states: torch.Tensor, + accept_lens: torch.Tensor, + next_token_ids: torch.Tensor, + rebound_spec_info, + verify_slots: Optional[torch.Tensor] = None, + ): + from types import SimpleNamespace + + # Shim the GenerationBatchResult _draft_extend_for_decode expects: it + # reads logits_output.hidden_states / accept_lens / next_token_ids and + # fills topk_p / topk_index / hidden_states / dsa_topk_indices in + # place on next_draft_input — here the ring-rebound spec_info. + shim = GenerationBatchResult( + logits_output=SimpleNamespace(hidden_states=verify_hidden_states), + accept_lens=accept_lens, + next_token_ids=next_token_ids, + next_draft_input=rebound_spec_info, + ) + # [PP+spec] Stage-boundary host sync (unconditional) — see the + # draftstep note for the rationale. + if _pp_sync_mask() & 0x40: # site7 replay-in + torch.cuda.current_stream().synchronize() + if _PP_SPEC_DEBUG: + logger.info( + f"[PPSPEC] replay-in pp_rank={get_pp_group().rank_in_group} " + f"hidden={tuple(verify_hidden_states.shape)} " + f"accept={tuple(accept_lens.shape)}/{accept_lens.tolist()[:8]} " + f"nti={tuple(next_token_ids.shape)} " + f"bs={len(batch.seq_lens)} seq_lens={batch.seq_lens.tolist()[:8]} " + f"bocl={tuple(batch.out_cache_loc.shape) if batch.out_cache_loc is not None else None} " + f"slots={tuple(verify_slots.shape) if verify_slots is not None else None}" + ) + if verify_slots is not None: + # [EXPERIMENT] PP+spec: the draft-extend commits the accepted tree + # rows into the verify-assigned draft-KV slots. On the last stage + # those are still on batch.out_cache_loc (in-forward window); here + # _forward_isolation already reverted them, so re-supply the + # ring-relayed slots explicitly. + batch.out_cache_loc = verify_slots + saved = {f: getattr(batch, f, None) for f in self._PP_REPLAY_RESTORE_FIELDS} + try: + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + ): + self.draft_worker._draft_extend_for_decode(batch, shim) + finally: + for f, v in saved.items(): + setattr(batch, f, v) + + def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput): + monkey_patch_torch_reductions() + named_tensors = MultiprocessingSerializer.deserialize( + recv_req.serialized_named_tensors[self.ps.tp_rank] + ) + success, message = ( + self.draft_worker.draft_runner.weight_updater.update_weights_from_tensor( + named_tensors=named_tensors, + load_format=recv_req.load_format, + ) + ) + if not success: + return success, message + + success, message = ( + self.target_worker.model_runner.weight_updater.update_weights_from_tensor( + named_tensors=named_tensors, + load_format=recv_req.load_format, + ) + ) + return success, message diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_ann_census.py b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_ann_census.py new file mode 100644 index 0000000..9ff4668 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_ann_census.py @@ -0,0 +1,31 @@ +import gzip, json, sys +from collections import defaultdict + +def census(path): + with gzip.open(path, 'rt') as f: + data = json.load(f) + ann = defaultdict(lambda: [0, 0.0]) + gloo = defaultdict(lambda: [0, 0.0, 0.0]) # name -> [n, total, max] + for e in data['traceEvents']: + if e.get('ph') != 'X': continue + nm = e.get('name', ''); cat = e.get('cat', ''); d = e.get('dur', 0) + if cat == 'user_annotation': + ann[nm][0] += 1; ann[nm][1] += d + elif nm.startswith('gloo:') and cat not in ('kernel',): + gloo[nm][2] = max(gloo[nm][2], d) + gloo[nm][0] += 1; gloo[nm][1] += d + print(f"== {path.split('/')[-1]} ==") + print(" user_annotations:") + for nm in sorted(ann, key=lambda x: -ann[x][1]): + n, t = ann[nm] + print(f" {t/1e3:9.1f}ms n={n:5d} {nm[:100]}") + print(" gloo ops:") + for nm in sorted(gloo, key=lambda x: -gloo[x][1]): + n, t, mx = gloo[nm] + print(f" {t/1e3:9.1f}ms n={n:5d} max={mx/1e3:8.1f}ms {nm}") + print() + +if __name__ == '__main__': + import glob + for p in sorted(glob.glob(sys.argv[1])): + census(p) diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_analyze.py b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_analyze.py new file mode 100644 index 0000000..c6a436b --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_analyze.py @@ -0,0 +1,104 @@ +import gzip, json, sys +from collections import defaultdict + +def merged(intervals): + ivs = sorted((s, e) for s, e in intervals if e > s) + out = [] + for s, e in ivs: + if out and s <= out[-1][1]: + if e > out[-1][1]: out[-1][1] = e + else: + out.append([s, e]) + return out + +def analyze(path, n_steps=12): + with gzip.open(path, 'rt') as f: + data = json.load(f) + ev = [e for e in data['traceEvents'] if e.get('ph') == 'X'] + kernels = [] # (ts, end, name) us + memcpy = [] + cpu_rt = defaultdict(lambda: [0, 0.0]) # cuda_runtime name -> [n, total_us] + cpu_sync_windows = [] # cudaStreamSynchronize (ts, end) + launches = [] + for e in ev: + cat = e.get('cat', '') + ts = e['ts']; dur = e.get('dur', 0) + if cat == 'kernel': + kernels.append((ts, ts + dur, e.get('name', ''))) + elif cat == 'gpu_memcpy': + memcpy.append((ts, ts + dur, e.get('name', ''))) + elif cat == 'cuda_runtime': + nm = e.get('name', '') + cpu_rt[nm][0] += 1; cpu_rt[nm][1] += dur + if nm == 'cudaStreamSynchronize': + cpu_sync_windows.append((ts, ts + dur)) + elif nm == 'cudaLaunchKernel': + launches.append((ts, dur)) + if not kernels: + print(f"{path}: no kernels"); return + t0 = min(k[0] for k in kernels); t1 = max(k[1] for k in kernels) + # long host-side ops (any ph=X outside cuda_runtime): gloo recv, work.wait, python fns + long_cpu = [] + for e in ev: + cat = e.get('cat', '') + if cat in ('cuda_runtime', 'kernel', 'gpu_memcpy'): continue + d = e.get('dur', 0) + if d > 1000: + long_cpu.append((e['ts'] - t0, d, e.get('name', '')[:110], cat)) + long_cpu.sort(key=lambda x: -x[1]) + span = t1 - t0 + busy_iv = merged([(k[0], k[1]) for k in kernels]) + busy = sum(e - s for s, e in busy_iv) + idle = span - busy + # gaps + gaps = [] + for (s1, e1), (s2, e2) in zip(busy_iv, busy_iv[1:]): + if s2 - e1 > 0: gaps.append((e1, s2 - e1)) + # attribute gaps >0.3ms: overlapping a cudaStreamSynchronize window? + sync_set = merged(cpu_sync_windows) + def overlaps_sync(gts): + for s, e in sync_set: + if s < gts and e > gts - 0: # any overlap + return True + return False + big = [(gs, gd) for gs, gd in gaps if gd > 300] + sync_attr = sum(gd for gs, gd in big if any(s < gs + gd and e > gs for s, e in sync_set)) + # kernel buckets + buckets = defaultdict(lambda: [0, 0.0]) + for ts, te, nm in kernels: + b = ('nccl_SendRecv' if 'SendRecv' in nm else + 'nccl_AllReduce' if 'AllReduce' in nm else + 'nccl_other' if 'ncclDevKernel' in nm else + 'mqa_logits' if 'mqa_logits' in nm else + 'sparse_mla' if 'sparse_mla' in nm else + 'cutlass_moe' if 'cutlass' in nm.lower() else + 'other') + buckets[b][0] += 1; buckets[b][1] += te - ts + mc = sum(1 for _ in memcpy); mt = sum(e - s for s, e, _ in memcpy) + rt = lambda n: cpu_rt.get(n, [0, 0.0]) + n_sync, sync_us = rt('cudaStreamSynchronize') + n_lk, lk_us = rt('cudaLaunchKernel') + n_ev, ev_us = rt('cudaEventSynchronize') + n_ecr, ecr_us = rt('cudaEventRecord') + print(f"== {path.split('/')[-1]} ==") + print(f" span {span/1e6:.3f}s n_steps~{n_steps} round~{span/1e3/n_steps:.1f}ms") + print(f" GPU-busy {busy/1e3:.1f}ms ({100*busy/span:.1f}%) idle {idle/1e3:.1f}ms ({100*idle/span:.1f}%) kernels n={len(kernels)}") + print(f" host cudaStreamSynchronize: n={n_sync} total={sync_us/1e3:.1f}ms (per-call avg {sync_us/max(n_sync,1)/1e3:.2f}ms)") + print(f" host cudaEventSynchronize: n={n_ev} total={ev_us/1e3:.1f}ms cudaEventRecord n={n_ecr}") + print(f" host cudaLaunchKernel: n={n_lk} cpu={lk_us/1e3:.1f}ms (avg {lk_us/max(n_lk,1):.0f}us) memcpy n={mc} gpu={mt/1e3:.1f}ms") + print(f" gaps>0.3ms: n={len(big)} total={sum(gd for _,gd in big)/1e3:.1f}ms top5={[f'{gd/1e3:.1f}' for _,gd in sorted(big,key=lambda x:-x[1])[:5]]}ms") + print(f" gap-us attributed to cudaStreamSynchronize overlap: {sync_attr/1e3:.1f}ms") + print(f" long host ops >1ms: n={len(long_cpu)} total={sum(d for _,d,_,_ in long_cpu)/1e3:.1f}ms; top10:") + for ts, d, nm, cat in long_cpu[:10]: + print(f" +{ts/1e3:9.1f}ms {d/1e3:7.1f}ms [{cat}] {nm}") + for b in sorted(buckets, key=lambda x: -buckets[x][1]): + n, d = buckets[b] + if d > 1000: + print(f" {b:14s} {d/1e3:9.1f}ms n={n:6d} ({100*d/span:.1f}% span, {100*d/busy:.1f}% busy)") + print() + +if __name__ == '__main__': + base = sys.argv[1] if len(sys.argv) > 1 else '/data/hf_models/ppmtp_profiles/M_decode' + import glob + for p in sorted(glob.glob(base + '/pm-*.trace.json.gz')): + analyze(p) diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode.sh new file mode 100644 index 0000000..db1193d --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# PP+MTP spec-decode round 采集（60.5, glm53-nvfp4:30000, TP4PP2+MTP 全 eager） +# 用法: bash prof_mtp_decode.sh +# 产出: /data/hf_models/ppmtp_profiles/M_decode/ 8 ranks x 12 steps trace +set -u +BASE=/data/hf_models/ppmtp_profiles +URL=http://127.0.0.1:30000 +mkdir -p "$BASE/M_decode" /root/bs_results /data/hf_models/bs_results + +health() { [ "$(curl -s -o /dev/null -m3 -w '%{http_code}' $URL/health)" = "200" ]; } +health || { echo "server not healthy"; exit 1; } + +SEED=6221 +curl -s -X POST $URL/flush_cache >/dev/null; sleep 3 + +D0=$(docker logs glm53-nvfp4 2>&1 | grep -c "Decode batch") +docker exec glm53-nvfp4 python3 -m sglang.bench_serving \ + --backend sglang --host 127.0.0.1 --port 30000 \ + --dataset-name random-ids --tokenizer /data/hf_models/GLM-5.3-NVFP4 \ + --num-prompts 16 --random-input-len 16384 --random-output-len 512 \ + --random-range-ratio 1.0 --max-concurrency 16 --temperature 0.0 \ + --warmup-requests 0 --seed $SEED \ + --output-file /data/hf_models/bs_results/prof_M_mtp_${SEED}.json \ + > /root/bs_results/prof_M_bench.log 2>&1 & +BPID=$! + +for i in $(seq 1 120); do + D=$(docker logs glm53-nvfp4 2>&1 | grep -c "Decode batch") + [ "$D" -gt "$D0" ] && break + sleep 2 +done +echo "[M] decode phase after ~$((i*2))s (decode-lines D0=$D0 -> D=$D)" + +curl -s -X POST $URL/start_profile -H 'Content-Type: application/json' \ + -d "{\"output_dir\":\"$BASE/M_decode\",\"num_steps\":12,\"activities\":[\"CPU\",\"GPU\"],\"with_stack\":false,\"record_shapes\":false,\"profile_prefix\":\"pm\"}" +echo; echo "[arm] 12 steps at $(date +%T)" + +wait $BPID; echo "[bench] exit=$? (数字仅参考, 采集期内)" +grep -E "Output token throughput|Mean TPOT" /root/bs_results/prof_M_bench.log | tail -2 + +n=0 +for i in $(seq 1 90); do + n=$(ls "$BASE/M_decode"/*.trace.json.gz 2>/dev/null | wc -l) + [ "$n" -ge 8 ] && { echo "[traces] $n files after ~$((i*5))s"; ls -la "$BASE/M_decode" | tail -10; exit 0; } + sleep 5 +done +echo "[traces] TIMEOUT only $n files"; ls -la "$BASE/M_decode" 2>/dev/null diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode2.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode2.sh new file mode 100644 index 0000000..f344ad8 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode2.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# PP+MTP spec-decode 稳态采集 v2：等全部 16 请求进入 decode（#running-req: 16 连续 5 行）后 arm。 +set -u +BASE=/data/hf_models/ppmtp_profiles +URL=http://127.0.0.1:30000 +mkdir -p "$BASE/M2_decode" /root/bs_results /data/hf_models/bs_results +health() { [ "$(curl -s -o /dev/null -m3 -w '%{http_code}' $URL/health)" = "200" ]; } +health || { echo "server not healthy"; exit 1; } + +SEED=6231 +curl -s -X POST $URL/flush_cache >/dev/null; sleep 3 + +D0=$(docker logs glm53-nvfp4 2>&1 | grep -c "#running-req: 16") +docker exec glm53-nvfp4 python3 -m sglang.bench_serving \ + --backend sglang --host 127.0.0.1 --port 30000 \ + --dataset-name random-ids --tokenizer /data/hf_models/GLM-5.3-NVFP4 \ + --num-prompts 16 --random-input-len 16384 --random-output-len 512 \ + --random-range-ratio 1.0 --max-concurrency 16 --temperature 0.0 \ + --warmup-requests 0 --seed $SEED \ + --output-file /data/hf_models/bs_results/prof_M2_mtp_${SEED}.json \ + > /root/bs_results/prof_M2_bench.log 2>&1 & +BPID=$! + +echo "[M2] waiting for steady decode (#running-req: 16 x5)..." +for i in $(seq 1 240); do + D=$(docker logs glm53-nvfp4 2>&1 | grep -c "#running-req: 16") + [ "$D" -ge $((D0 + 5)) ] && break + sleep 2 +done +echo "[M2] steady decode after ~$((i*2))s (D0=$D0 -> D=$D)" + +curl -s -X POST $URL/start_profile -H 'Content-Type: application/json' \ + -d "{\"output_dir\":\"$BASE/M2_decode\",\"num_steps\":12,\"activities\":[\"CPU\",\"GPU\"],\"with_stack\":false,\"record_shapes\":false,\"profile_prefix\":\"pm2\"}" +echo; echo "[arm] 12 steps at $(date +%T)" + +wait $BPID; echo "[bench] exit=$?" +n=0 +for i in $(seq 1 90); do + n=$(ls "$BASE/M2_decode"/*.trace.json.gz 2>/dev/null | wc -l) + [ "$n" -ge 8 ] && { echo "[traces] $n files after ~$((i*5))s"; ls -la "$BASE/M2_decode" | tail -10; exit 0; } + sleep 5 +done +echo "[traces] TIMEOUT only $n files"; ls -la "$BASE/M2_decode" 2>/dev/null diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode3.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode3.sh new file mode 100644 index 0000000..25ced5a --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/prof_mtp_decode3.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# PP+MTP 稳态采集 v3: 16k in / 2048 out 拉长 decode 段, 等 running-req>=12 连续5行后 arm 12 步。 +set -u +BASE=/data/hf_models/ppmtp_profiles +URL=http://127.0.0.1:30000 +mkdir -p "$BASE/M3_decode" /root/bs_results /data/hf_models/bs_results +rmdir "$BASE/M2_decode" 2>/dev/null +health() { [ "$(curl -s -o /dev/null -m3 -w '%{http_code}' $URL/health)" = "200" ]; } +health || { echo "server not healthy"; exit 1; } + +SEED=6241 +curl -s -X POST $URL/flush_cache >/dev/null; sleep 3 + +D0=$(docker logs glm53-nvfp4 2>&1 | grep -cE '#running-req: 1[2-9],|#running-req: 16,') +docker exec glm53-nvfp4 python3 -m sglang.bench_serving \ + --backend sglang --host 127.0.0.1 --port 30000 \ + --dataset-name random-ids --tokenizer /data/hf_models/GLM-5.3-NVFP4 \ + --num-prompts 16 --random-input-len 16384 --random-output-len 2048 \ + --random-range-ratio 1.0 --max-concurrency 16 --temperature 0.0 \ + --warmup-requests 0 --seed $SEED \ + --output-file /data/hf_models/bs_results/prof_M3_mtp_${SEED}.json \ + > /root/bs_results/prof_M3_bench.log 2>&1 & +BPID=$! + +echo "[M3] waiting for high-bs decode (running-req>=12 x5)..." +for i in $(seq 1 300); do + D=$(docker logs glm53-nvfp4 2>&1 | grep -cE '#running-req: 1[2-9],|#running-req: 16,') + [ "$D" -ge $((D0 + 5)) ] && break + sleep 2 +done +echo "[M3] high-bs decode after ~$((i*2))s (D0=$D0 -> D=$D)" + +curl -s -X POST $URL/start_profile -H 'Content-Type: application/json' \ + -d "{\"output_dir\":\"$BASE/M3_decode\",\"num_steps\":12,\"activities\":[\"CPU\",\"GPU\"],\"with_stack\":false,\"record_shapes\":false,\"profile_prefix\":\"pm3\"}" +echo; echo "[arm] 12 steps at $(date +%T)" + +# 不等 bench 完成: arm 后 12 步 ~4s + flush ~11s, 之后再等 bench +sleep 40 +n=0 +for i in $(seq 1 60); do + n=$(ls "$BASE/M3_decode"/*.trace.json.gz 2>/dev/null | wc -l) + [ "$n" -ge 8 ] && { echo "[traces] $n files after ~$((i*5))s"; break; } + sleep 5 +done +[ "$n" -ge 8 ] || { echo "[traces] only $n files"; ls -la "$BASE/M3_decode" 2>/dev/null; } +kill $BPID 2>/dev/null +wait $BPID 2>/dev/null +echo DONE diff --git a/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/race_run.sh b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/race_run.sh new file mode 100644 index 0000000..4c4daa4 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_pro6000d_sglang_ppmtp_deepdive/scripts/race_run.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# 竞态二分单点运行器: race_run.sh