From e1719bd575a5d86cb4fe1c05fde333740f74a782 Mon Sep 17 00:00:00 2001 From: Zhiyi Hong <2497491955@qq.com> Date: Sat, 1 Aug 2026 15:29:09 +0800 Subject: [PATCH] [Docs] finalize Phase 2.5 RDMA demand model --- README.md | 4 + .../6000D双机通信与NCCL术语入门.html | 20 + .../phase2_5_code.html | 167 +++ .../phase2_5_exp.html | 166 +++ .../phase2_exp.html | 1 + .../commands/confirm.cmd.txt | 1 + .../commands/scout.cmd.txt | 1 + .../confirm/bench_summary.csv | 7 + .../confirm/case_rdma_summary.csv | 25 + .../confirm/case_windows.csv | 7 + .../confirm/collector_status.csv | 37 + .../confirm/service/head_nccl_transport.log | 1012 +++++++++++++++++ .../confirm/service/head_server_cmd.txt | 1 + .../confirm/service/worker_nccl_transport.log | 918 +++++++++++++++ .../confirm/service/worker_server_cmd.txt | 1 + .../rdma_case_metrics.csv | 12 + .../rdma_demand_model.json | 205 ++++ .../rdma_demand_report.md | 37 + .../recommendation.env | 1 + .../run_manifest.txt | 13 + .../scenarios/confirm.tsv | 4 + .../scenarios/scout.tsv | 6 + .../scout/bench_summary.csv | 6 + .../scout/case_rdma_summary.csv | 21 + .../scout/case_windows.csv | 6 + .../scout/collector_status.csv | 37 + .../scout/service/head_nccl_transport.log | 1012 +++++++++++++++++ .../scout/service/head_server_cmd.txt | 1 + .../scout/service/worker_nccl_transport.log | 918 +++++++++++++++ .../scout/service/worker_server_cmd.txt | 1 + .../推理优化计划.html | 20 +- 31 files changed, 4661 insertions(+), 7 deletions(-) create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/phase2_5_code.html create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/phase2_5_exp.html create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/confirm.cmd.txt create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/scout.cmd.txt create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/bench_summary.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_rdma_summary.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_windows.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/collector_status.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_nccl_transport.log create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_server_cmd.txt create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_nccl_transport.log create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_server_cmd.txt create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_case_metrics.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_model.json create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_report.md create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/recommendation.env create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/run_manifest.txt create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/confirm.tsv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/scout.tsv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/bench_summary.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_rdma_summary.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_windows.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/collector_status.csv create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_nccl_transport.log create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_server_cmd.txt create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_nccl_transport.log create mode 100644 docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_server_cmd.txt diff --git a/README.md b/README.md index e001456..3c0913a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # sskj — 多平台大模型推理性能基准测试项目 +> **更新（2026-08-01 15:24:52 CST）** +> +> 完成 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 2.5 RDMA 需求建模。正式 Run `dsv4pro-phase2_5-20260801-130007` 完成 Scout 5/5 与 Confirm 6/6；64K Prefill 在 C=16 已进入约 2,984 input tok/s、79.90 Gbit/s/rail 的平台，C=32/64 不再显著增长。拟合通信强度为 3.332 MB/input-token/rail，单 Rail 400G 需约 15,006 input tok/s，约为当前平台的 5 倍，因此当前是模型计算/实现吞吐先饱和，不是 RDMA 先饱和。新增 `phase2_5_exp.html`、`phase2_5_code.html`、精简证据集和可复用的模型部署 RDMA 需求评估流程；实验结束后双节点容器与 16 张 GPU 已清理。 +> > **更新（2026-08-01 02:40:00 CST）** > > 新增 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 2.5 RDMA 需求建模唯一入口。实验保持现有 TP16/EP2 服务参数不变，先以 `64K -> 1` 的 C=1/4/16/32/64 建立 Input TPS 与每 Rail HCA 带宽关系，再自动选择平台前、拐点和最大稳定并发，对 `64K -> 1K` 重复确认。结果将给出每 Token 跨机字节数、400G 所需 Token TPS、并发饱和曲线和“模型计算先饱和还是 RDMA 先饱和”的机器可读结论；正式结果尚未生成，因此暂不创建 Phase 2.5 HTML 档案。 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/6000D双机通信与NCCL术语入门.html b/docs/dsv4pro_pro6000d_2node_sglang/6000D双机通信与NCCL术语入门.html index e24993e..8a7dbd2 100644 --- a/docs/dsv4pro_pro6000d_2node_sglang/6000D双机通信与NCCL术语入门.html +++ b/docs/dsv4pro_pro6000d_2node_sglang/6000D双机通信与NCCL术语入门.html @@ -547,6 +547,26 @@ mlx5_3 port 1 ==> eth3 (Up) 查看系统或进程在各 NUMA 节点上的内存分布。 Phase 2 每 5 秒保存结构化 Node0/Node1 MiB，寻找跨 NUMA 内存放置。 + + HCA Counter + 网卡硬件维护的发送、接收、等待、丢弃和错误累计计数器。 + Phase 2.5 用 mlx5_0/mlx5_3 的 counter 差值计算正式 benchmark 窗口内的 RDMA Gbit/s。 + + + bytes/input-token/rail + 模型每处理一个输入 token，平均要在一条 Rail 上发送的字节数。 + 当前 DSV4-Pro TP16/EP2 Scout 拟合为约 3.332 MB/token/rail；换模型或并行策略必须重新标定。 + + + 带宽平台 / 拐点 + 继续增加并发后，吞吐与网络带宽都几乎不再增长的位置。 + Phase 2.5 以相邻点的 Input TPS 和 Rail Mean 增益同时低于 5% 判断，当前拐点为 C=16。 + + + 渐近线 / 饱和上限 + 饱和曲线在并发继续增大时逼近、但不会明显超过的预测上限。 + 当前 64K Prefill 的拟合上限约 80.32 Gbit/s/rail，表示模型产流量上限，不表示网卡硬件只能跑 80G。 + diff --git a/docs/dsv4pro_pro6000d_2node_sglang/phase2_5_code.html b/docs/dsv4pro_pro6000d_2node_sglang/phase2_5_code.html new file mode 100644 index 0000000..fbb01f6 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/phase2_5_code.html @@ -0,0 +1,167 @@ + + + + + + Phase 2.5 Code：DSV4-Pro 双机 SGLang RDMA 需求建模 + + + +
+
+

STANDALONE CODE WALKTHROUGH / PHASE 2.5

+

DSV4-Pro 双机 Pro6000D SGLang RDMA 需求建模：代码详解

+
实现提交：c5fa700c50c0　正式 Run：dsv4pro-phase2_5-20260801-130007　唯一入口：run_rdma_demand_modeling.sh all
+
+
+
+

返回推理优化主计划 · 打开 Phase 2.5 实验档案

+
边界：Phase 2.5 不复制模型服务和采集器。它复用 Phase 1 的双机 TP16 服务/benchmark 与 Phase 2 的精确窗口、GPU/RDMA 采集能力，只新增“并发 Scout → 自动选点 → 业务 OSL Confirm → 需求拟合”这一层编排和分析。
+ +

1. 文件职责与调用关系

+ + + + + + + + + + +
文件职责调用关系
config.env定义 ISL/OSL、Scout 并发、重复次数、平台阈值和 400G 目标被唯一入口 source
run_rdma_demand_modeling.sh生成场景、调用 Phase 2、串行运行 Scout/Confirm、清理服务唯一人工入口
rdma_demand_model.py对齐 benchmark/HCA 窗口，计算 bytes/token，拟合平台并生成报告Scout 后选点；Confirm 后最终汇总
test_rdma_demand_model.py覆盖 HCA counter 单位、平台选择、线性换算和拟合输出本地/CI 回归测试
Phase 2 run_hardware_contention_attribution.sh启动服务、采集 Head/Worker、切正式测量窗由 Phase 2.5 以环境变量调用
Phase 1 run_quick_map.sh双机服务启停与 SGLang benchmark由 Phase 2 内部复用
+ 
run_rdma_demand_modeling.sh all
+  ├─ validate_config + write_manifest
+  ├─ run_scout
+  │    ├─ write_scenario_file(64K→1, C=1/4/16/32/64)
+  │    ├─ Phase 2 all（服务 + 18 个采集器 + 5 Case）
+  │    └─ rdma_demand_model.py scout → recommendation.env
+  ├─ run_confirm
+  │    ├─ 读取自动选择的 C=4/16/64
+  │    ├─ Phase 2 all（重启服务 + 18 个采集器 + 每点 2 次）
+  │    └─ rdma_demand_model.py final
+  └─ Phase 2 stop → 双节点清理
+ +

2. 配置层

+

config.env:5-7 通过相对路径找到 Phase 1/2，不依赖执行命令所在目录。config.env:10-16 定义 64K Scout 与 1K Confirm；18-21 定义 5% 平台阈值、400G 物理目标和 360G 实用目标。

+ 
ISL=65536
+SCOUT_OSL=1
+CONFIRM_OSL=1024
+SCOUT_CONCURRENCIES="1 4 16 32 64"
+SCOUT_REPETITIONS=1
+CONFIRM_REPETITIONS=2
+PLATEAU_GAIN_PCT=5
+TARGET_RAIL_GBPS=400
+PRACTICAL_RAIL_GBPS=360
+

SAMPLE_INTERVAL_S=1 只决定 HCA/GPU 时间序列分辨率；SCENARIO_TIMEOUT_S=7200 是单个 benchmark 的保护上限，不是期望耗时。

+ +

3. Shell 唯一入口

+

3.1 参数检查与场景生成

+

run_rdma_demand_modeling.sh:42-71 fail-fast 检查依赖脚本、整数参数和并发列表。73-100 生成 Phase 2 能读取的 TSV，并为每个形状生成稳定的 case id。

+

3.2 复用 Phase 2，而不是复制采集代码

+

103-134 构造一个数组命令，把场景、Case、重复次数和采样周期作为环境变量传给 Phase 2。它显式关闭 mixed case 与通信 microbenchmark，因为 Phase 2.5 只测模型 RDMA 需求，不重复已完成的硬件基线。

+ 
RUN_MIXED_CASE=0
+RUN_COMMUNICATION_BASELINE=0
+SCENARIO_FILE=.../scout.tsv
+FIXED_CASE_IDS=rdma_scout_...
+bash run_hardware_contention_attribution.sh all
+

3.3 两阶段控制流

+

136-152 跑完 Scout 后立即调用 Python，并写出 recommendation.env；154-180 读取推荐并发，生成 64K→1K Confirm。217-225 的 run_all 严格串行执行，异常信号触发 stop 清理。

+
为什么服务会启动两次：Scout 结束后 Phase 2 会清理服务；Confirm 使用全新的 Prefix Cache、采集器和服务生命周期，避免 Scout 状态污染确认结果。
+ +

4. Python 如何从计数器变成需求模型

+

4.1 精确时间窗与 HCA 单位

+

rdma_demand_model.py:87-115 以 (case_id,repetition) 对齐 benchmark、窗口和 RDMA 汇总。117-147 在正式窗口内计算相邻 HCA counter 的速率；IB port_*_data 单位是 4-octet，因此必须乘 4，再乘 8 转为 bit/s。

+ 
gbps = (counter_delta × 4 bytes × 8 bits) / duration_s / 1e9
+

4.2 单 Case 指标

+

171-281 汇总四条观测边（Head/Worker × 两个 HCA）的 Rail Mean/P95/Max、双 Rail 单向合计、Rail 不均衡、错误计数和 GPU 利用率。通信强度按每条 Rail 平均发送字节计算：

+ 
bytes_per_input_token_per_rail
+= mean(head/worker × mlx5_0/mlx5_3 xmit_bytes)
+  / total_input_tokens
+

这里不把 TX+RX 相加，因为那会把同一份跨机数据重复计数。

+ +

4.3 平台、拐点与自动选点

+

362-392 比较相邻并发点。只有 Rail Mean 与 Input TPS 增益同时低于 5%，当前点才是平台候选。随后选择平台前一点、平台点和最高稳定点；本 Run 得到 4 16 64。

+ 
if bandwidth_gain < 5% and input_tps_gain < 5%:
+    plateau_c = current_concurrency
+ +

4.4 线性通信强度与饱和曲线

+

352-360 用过原点线性斜率拟合 rail_gbps/input_tps，再还原为 bytes/token。318-350 用双曲线 B(C)=B∞×C/(K+C) 拟合并发饱和曲线；394-447 组合两者，判断模型计算或网络谁先到平台。

+ 
required_input_tps
+= target_rail_gbps / linear_gbps_per_input_tps
+
+if fitted_bandwidth_asymptote < 360:
+    verdict = COMPUTE_OR_MODEL_THROUGHPUT_LIMITED_BEFORE_RDMA_SATURATION
+

本 Run 的 RMSE 为 0.277 Gbit/s，五个 Scout 点与饱和曲线贴合良好；拟合上限 80.32 Gbit/s，与 C=16/32/64 的 79.90/79.93/79.97 一致。

+ +

5. 输出文件如何阅读

+ + + + + + + + + + + +
输出用途
rdma_case_metrics.csv每次重复的 benchmark + GPU + Rail 对齐数据，是审计主表
rdma_demand_model.json完整拟合参数、平台点、目标 TPS 和最终 verdict
rdma_demand_report.md面向人的 Scout/Confirm 摘要
recommendation.envShell 可直接 source 的 Confirm 并发列表
{scout,confirm}/case_windows.csv每个正式 benchmark 的精确起止时间
{scout,confirm}/{head,worker}/rdma.csv原始 HCA counter 时间序列，仅保留在服务器完整结果中
commands/*.cmd.txt实际传给 Phase 2 的完整可复现命令
+ +

6. 测试与验收门槛

+ + +

7. 行号索引

+ + + + + + + + + + + + + + + +
功能文件与行
配置与路径config.env:3-36
校验与场景生成run_rdma_demand_modeling.sh:42-100
Phase 2 调用run_rdma_demand_modeling.sh:103-134
Scout / Confirmrun_rdma_demand_modeling.sh:136-180
唯一 all 与清理run_rdma_demand_modeling.sh:206-264
HCA interval raterdma_demand_model.py:117-147
Case 对齐汇总rdma_demand_model.py:171-281
饱和拟合rdma_demand_model.py:318-350
平台选点rdma_demand_model.py:362-392
需求模型与 verdictrdma_demand_model.py:394-447
报告输出rdma_demand_model.py:449-517
+ +

8. 复用时必须重新标定的边界

+

这套代码可复用，但 3.332 MB/token/rail 不是通用常数。换模型、量化、TP/EP、节点切分、backend、Prefill/Decode 形状或 Prefix Cache 策略后，都必须重新跑 Scout。代码输出的是“当前部署实现的经验模型”，不是由参数量单独推导出的理论通信量。

+
+ + diff --git a/docs/dsv4pro_pro6000d_2node_sglang/phase2_5_exp.html b/docs/dsv4pro_pro6000d_2node_sglang/phase2_5_exp.html new file mode 100644 index 0000000..0cd60c6 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/phase2_5_exp.html @@ -0,0 +1,166 @@ + + + + + + Phase 2.5：DSV4-Pro 双机 Pro6000D SGLang RDMA 需求建模 + + + +
+
+

FINAL EXPERIMENT RECORD / PHASE 2.5

+

DeepSeek-V4-Pro 双机 Pro6000D SGLang：RDMA 需求建模与并发拐点

+
+ Run：dsv4pro-phase2_5-20260801-130007 + 拓扑：TP16 / EP2 / 双 Rail RoCE + 完成：2026-08-01 15:14:11 CST +
+
+
+
+ 返回推理优化主计划 + 打开 Phase 2.5 代码详解 + +

阶段已完成。正式 Run 用时 2 小时 14 分 04 秒；Scout 5/5、Confirm 6/6 成功，两阶段各 18/18 个采集器正常启停。所有测量窗 RDMA 错误增量为 0，结束后两节点容器和 16 张 GPU 均已清理。

+ +

1. 要回答的问题

+

Phase 2 只看到代表负载约 83.5 Gbit/s/rail，不能判断继续增加并发是否会逼近 400G。Phase 2.5 专门回答三个问题：

+
    +
  1. 固定模型、TP/EP 和输入形状后，Input TPS 与每 Rail RDMA 带宽是什么关系？
    2. +
  2. 并发增加到哪里后，模型吞吐和 RDMA 带宽不再增长？
    3. +
  3. 要达到 400G，需要怎样的 Input TPS；当前瓶颈先出现在模型计算还是网络？
    4. +
+ +

2. 实验设计

+ + + + + + +
阶段请求形状并发重复目的
Scout64K → 11 / 4 / 16 / 32 / 641隔离 Prefill，找吞吐与带宽平台
Confirm64K → 1K自动选择 4 / 16 / 642验证真实长输出不会推翻需求模型
+

服务参数沿用 Phase 1/2：SGLang nightly、TP16、EP2、双 Rail mlx5_0/mlx5_3、NET/IB + GDRDMA。每个 Case 使用冷 Prefix，并按 benchmark 正式测量窗口切片 HCA Counter。

+ +

3. 实际启动命令

+

只在 Head 174.1.51.5 执行，不需要 source 或 conda activate：

+ 
cd /data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling
+
+RUN_ID=dsv4pro-phase2_5-20260801-130007
+tmux new-session -d -s dsv4pro-phase2_5 \
+  "RUN_ID=${RUN_ID} bash run_rdma_demand_modeling.sh all \
+   2>&1 | tee /data/hzy/${RUN_ID}.log"
+
+tmux attach -t dsv4pro-phase2_5
+

实际展开后的 Scout/Confirm 命令分别保存在结果目录的 commands/scout.cmd.txt 与 commands/confirm.cmd.txt。

+ +

4. Scout 结果：并发 16 已进入平台

+ + + + + + + + + +
CInput TPSRail MeanRail P95Rail Max双 Rail 单向合计MB/input-token/railGPU Util
12,709.6470.97 Gbit/s85.6086.81141.943.13893.75%
42,930.4978.27 Gbit/s86.8288.86156.553.29897.45%
162,983.7779.90 Gbit/s86.4288.20159.793.33699.33%
322,983.9279.93 Gbit/s86.2388.22159.863.34499.49%
642,991.2879.97 Gbit/s85.9388.32159.953.34099.64%
+

观察结论：C=16→32 的 Input TPS 只增长 0.005%，Rail Mean 只增长 0.040%；C=32→64 也仅增长 0.247% / 0.057%。并发 16 已是平台拐点，继续加到 64 只会增加排队和 TTFT，不会增加网络压力。

+

证据：/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/rdma_case_metrics.csv；原始 HCA 数据位于同一 Run 的 scout/head/rdma.csv 与 scout/worker/rdma.csv，精确窗口位于 scout/case_windows.csv。

+ +

5. Confirm 结果：加入 1K Decode 后仍由计算先饱和

+ + + + + + + +
C重复Input TPSOutput TPSRail MeanRail P95双 Rail单向合计TTFT P95TPOT P95
422,072.2432.3857.33 Gbit/s86.28114.6686.36 s95.63 ms
1622,582.1340.3571.64 Gbit/s86.30143.29336.85 s356.24 ms
6422,588.9640.4572.19 Gbit/s86.69144.381,506.50 s418.58 ms
+

C=16→64 的 Input TPS 仅增长 0.26%，Rail Mean 仅增长 0.76%，但 TTFT P95 从 336.85 秒升至 1,506.50 秒。对于 64K→1K，最大有意义并发仍约为 16；C=64 是容量压力点，不是推荐服务点。

+

证据：同一 Run 的 confirm/bench_summary.csv、confirm/case_rdma_summary.csv、confirm/case_windows.csv；两轮逐点数据在顶层 rdma_case_metrics.csv。

+ +

6. 400G 能否被模型负载打满

+

Scout 的线性比例为：

+ 
每 Rail 带宽（Gbit/s）
+≈ Input TPS × 3.332 MB/input-token/rail × 8 ÷ 1e9
+ + + + + + + +
目标口径需要的 Input TPS当前约 2,991 TPS 的差距
单 Rail 400G15,006 tok/s约 5.02×
单 Rail 360G（90% 实用线）13,505 tok/s约 4.51×
双 Rail 单向合计 400G7,503 tok/s约 2.51×
+

拟合得到当前模型负载的单 Rail 渐近上限约 80.32 Gbit/s，即物理 400G 的约 20.1%。瞬时 Max 也只有 89.53 Gbit/s。结论不是“网络只能跑 80G”，而是当前 DSV4-Pro TP16/EP2 实现最多只能产生约 80G/rail 的持续 RDMA 流量；Phase 2 的 NCCL microbenchmark 已证明链路本身能达到更高通信带宽。

+

口径提醒：400G 是每条 Rail 的线速；双 Rail 单向总量是两条 Rail 的 TX 之和。不要把 TX 与 RX 相加后声称打满，也不要把 NCCL busbw GB/s 与 HCA Gbit/s 直接比较。

+ +

7. 一套可复用的 RDMA 需求评估方法

+
    +
  1. 固定部署变量。记录模型版本、精度/量化、TP/EP/PP/DP、节点数、Attention/MoE backend、chunked prefill 和网卡拓扑。任一项变化都要重新标定。
    2. +
  2. 先选 Prefill Scout。固定 ISL，OSL=1，取稀疏并发点如 1/4/16/32/64；每点清 Prefix Cache，并保证请求文本实际达到目标 token 数。
    3. +
  3. 对齐正式测量窗。从 benchmark 的 main-run 起止时间切片 Head/Worker 的 mlx5_* HCA Counter，不能用整个进程寿命，也不能只看 sar eth*。
    4. +
  4. 计算通信强度。bytes_per_input_token_per_rail = rail_xmit_bytes / total_input_tokens。这是该模型与并行策略下“每处理一个输入 token，要在一条 Rail 发送多少字节”。
    5. +
  5. 找并发平台。同时观察 Input TPS 和 Rail Mean；连续一点的增益都低于阈值（本实验 5%）时，记为拐点。最大 C 不等于最大有效 C。
    6. +
  6. 推导目标吞吐。required_input_tps = target_rail_gbps × 1e9 / (bytes_per_token × 8)。若模型的实测/拟合 TPS 上限远低于该值，网络不会先饱和。
    7. +
  7. 用业务 OSL 复测。在平台前、拐点、最高压力点各重复至少两次，确认 Decode、KV Cache 和调度没有改变结论。
    8. +
  8. 最后做链路对照。模型负载未打满时，用 NCCL microbenchmark 验证网络能力，把“模型产流量不足”与“网络本身跑不满”分开。
    9. +
+ +

7.1 哪些变量会改变 bytes/token 与平台

+ + + + + + + + + + +
变量可能改变的原因
模型架构与层数每 token 触发的 TP collective、MoE dispatch/combine 和激活尺寸不同
TP / EP / PP / DP通信参与 rank、跨机边界、collective 类型和频率改变
Prefill / Decode、ISL / OSL计算强度、chunk 调度、KV 访问和 collective 消息粒度不同
并发与 batch决定 kernel/batch 效率和 Input TPS；超过平台后只增加排队
量化与 backend改变计算速度；通信字节可能不同比例变化，因此会移动“计算先饱和还是网络先饱和”的边界
Prefix Cache命中会绕过大量 Prefill，必须单独作为另一类业务场景建模
+ +

8. 最终结论

+

在两台 Pro6000D、DSV4-Pro、SGLang TP16/EP2 的当前实现中，RDMA 不是吞吐瓶颈。64K Prefill 在 C=16 已达到约 3K input tok/s 和 80 Gbit/s/rail 的平台；继续增加并发到 64 不会显著增加吞吐或带宽，只会令 TTFT 急剧上升。要打满单 Rail 400G，模型侧 Input TPS 需提高到约 15K，约为当前上限 5 倍。因此后续优化应先看 GPU Kernel、MoE/Attention 执行和 rank 同步，而不是扩容计算网。

+ +

9. 证据与清理

+ +

Worker 在最后一个 Case 完成后随 Head 主动关闭进程组出现 Gloo peer-close Traceback；它发生在测量结束与结果落盘之后，不是实验失败。最终 tmux、服务容器和 GPU 进程均已退出。

+
+ + diff --git a/docs/dsv4pro_pro6000d_2node_sglang/phase2_exp.html b/docs/dsv4pro_pro6000d_2node_sglang/phase2_exp.html index 88a8cca..32b87e1 100644 --- a/docs/dsv4pro_pro6000d_2node_sglang/phase2_exp.html +++ b/docs/dsv4pro_pro6000d_2node_sglang/phase2_exp.html @@ -805,6 +805,7 @@ tmux new-session -d -s dsv4pro-phase2 \

返回 Phase 1 实验档案

+

继续 Phase 2.5 RDMA 需求建模

返回推理优化主计划

diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/confirm.cmd.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/confirm.cmd.txt new file mode 100644 index 0000000..808a1bd --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/confirm.cmd.txt @@ -0,0 +1 @@ +env DRY_RUN=0 RUN_ID=confirm RESULT_BASE=/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007 SCENARIO_FILE=/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scenarios/confirm.tsv FIXED_CASE_IDS=rdma_confirm_i65536_o1024_c4\,rdma_confirm_i65536_o1024_c16\,rdma_confirm_i65536_o1024_c64 RUN_MIXED_CASE=0 RUN_COMMUNICATION_BASELINE=0 NUM_REPETITIONS=2 SAMPLE_INTERVAL_S=1 IDLE_BASELINE_S=5 POST_RUN_COOLDOWN_S=5 CASE_COOLDOWN_S=3 COLLECTOR_MAX_SECONDS=14400 SCENARIO_TIMEOUT_S=7200 bash /data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_hardware_contention_attribution/run_hardware_contention_attribution.sh all diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/scout.cmd.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/scout.cmd.txt new file mode 100644 index 0000000..a37f376 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/commands/scout.cmd.txt @@ -0,0 +1 @@ +env DRY_RUN=0 RUN_ID=scout RESULT_BASE=/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007 SCENARIO_FILE=/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scenarios/scout.tsv FIXED_CASE_IDS=rdma_scout_i65536_o1_c1\,rdma_scout_i65536_o1_c4\,rdma_scout_i65536_o1_c16\,rdma_scout_i65536_o1_c32\,rdma_scout_i65536_o1_c64 RUN_MIXED_CASE=0 RUN_COMMUNICATION_BASELINE=0 NUM_REPETITIONS=1 SAMPLE_INTERVAL_S=1 IDLE_BASELINE_S=5 POST_RUN_COOLDOWN_S=5 CASE_COOLDOWN_S=3 COLLECTOR_MAX_SECONDS=14400 SCENARIO_TIMEOUT_S=7200 bash /data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_hardware_contention_attribution/run_hardware_contention_attribution.sh all diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/bench_summary.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/bench_summary.csv new file mode 100644 index 0000000..28db272 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/bench_summary.csv @@ -0,0 +1,7 @@ +run_id,suite,case_id,role,stage,repetition,isl,osl,concurrency,num_prompts,warmup_requests,status,error_type,exit_code,started_at,ended_at,elapsed_s,measurement_started_at,measurement_ended_at,measurement_duration_s,measurement_window_source,completed,failed,duration_s,actual_concurrency,peak_concurrent_requests,total_input_tokens,total_output_tokens,request_throughput,input_token_throughput,output_token_throughput,total_token_throughput,peak_output_token_throughput,e2e_mean_ms,e2e_p50_ms,e2e_p95_ms,e2e_p99_ms,ttft_mean_ms,ttft_p50_ms,ttft_p95_ms,ttft_p99_ms,tpot_mean_ms,tpot_p50_ms,tpot_p95_ms,tpot_p99_ms,itl_mean_ms,itl_p50_ms,itl_p95_ms,itl_p99_ms,bench_file,bench_log,phase2_bench_run +fixed_rdma_confirm_i65536_o1024_c16,fixed,rdma_confirm_i65536_o1024_c16,,rdma_confirm,1,65536,1024,16,16,0,COMPLETED,,0,2026-08-01T14:03:43+0800,2026-08-01T14:11:03+0800,440.0,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,406.0844679859583,bench_main_marker_plus_duration,16,0,406.0844679859583,15.997723352218085,,1048576,16384,0.03940066971621592,2582.1622905219265,40.3462857894051,2622.5085763113316,,406026.68602950143,406024.73604350234,406053.3737935184,406053.5528299224,189372.86670175308,189251.9370575028,336778.7128902273,348624.74093402264,211.7828145921294,211.89852223703926,356.41907673483706,369.22852823690624,211.7827865053078,53.14174899831414,53.806967759737745,55.23117201111745,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c16/cases/rdma_confirm_i65536_o1024_c16/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c16/cases/rdma_confirm_i65536_o1024_c16/rep1/bench.log,fixed_rdma_confirm_i65536_o1024_c16 +fixed_rdma_confirm_i65536_o1024_c16,fixed,rdma_confirm_i65536_o1024_c16,,rdma_confirm,2,65536,1024,16,16,0,COMPLETED,,0,2026-08-01T14:11:03+0800,2026-08-01T14:18:24+0800,441.0,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,406.09330375201534,bench_main_marker_plus_duration,16,0,406.09330375201534,15.997232835777202,,1048576,16384,0.03939981243761298,2582.1061079114043,40.34540793611569,2622.45151584752,,406023.07082318654,406021.80599197163,406049.04031673505,406056.0000617319,189594.6209833055,189467.45998901315,336924.2522477143,348614.9598911492,211.56251206244485,211.68662201320043,356.06784952074946,368.88221031313344,211.56248251307755,53.15867651370354,54.19228565006051,55.67694299679715,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c16/cases/rdma_confirm_i65536_o1024_c16/rep2/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c16/cases/rdma_confirm_i65536_o1024_c16/rep2/bench.log,fixed_rdma_confirm_i65536_o1024_c16 +fixed_rdma_confirm_i65536_o1024_c4,fixed,rdma_confirm_i65536_o1024_c4,,rdma_confirm,1,65536,1024,4,4,0,COMPLETED,,0,2026-08-01T13:58:15+0800,2026-08-01T14:00:56+0800,161.0,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,127.71871886099689,bench_main_marker_plus_duration,4,0,127.71871886099689,3.9987289259054917,,262144,4096,0.031318823393095684,2052.5104098899187,32.07047515452998,2084.5808850444487,,127678.13387226488,127678.37510351092,127682.15249372879,127682.41568037542,58779.63522000937,59286.87346848892,87520.75181677355,89912.75672257761,67.34946104814811,66.85386279083285,96.14249014501952,98.71897348003505,67.3494309836352,36.35076849604957,36.65076588222291,37.22598696593195,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c4/cases/rdma_confirm_i65536_o1024_c4/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c4/cases/rdma_confirm_i65536_o1024_c4/rep1/bench.log,fixed_rdma_confirm_i65536_o1024_c4 +fixed_rdma_confirm_i65536_o1024_c4,fixed,rdma_confirm_i65536_o1024_c4,,rdma_confirm,2,65536,1024,4,4,0,COMPLETED,,0,2026-08-01T14:00:57+0800,2026-08-01T14:03:36+0800,159.0,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,125.30951252899831,bench_main_marker_plus_duration,4,0,125.30951252899831,3.9986913595171454,,262144,4096,0.03192096050229504,2091.9720674784076,32.68706355435012,2124.6591310327576,,125268.51625375275,125267.95494600083,125274.71757131862,125275.15292868251,56944.67479274317,57454.862895508995,85207.63279019155,87584.75151082734,66.7877238133036,66.2884575273625,95.11951526871552,97.68299949875195,66.78769251954503,36.23565100133419,36.565644716029055,37.02181185770315,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c4/cases/rdma_confirm_i65536_o1024_c4/rep2/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c4/cases/rdma_confirm_i65536_o1024_c4/rep2/bench.log,fixed_rdma_confirm_i65536_o1024_c4 +fixed_rdma_confirm_i65536_o1024_c64,fixed,rdma_confirm_i65536_o1024_c64,,rdma_confirm,1,65536,1024,64,64,0,COMPLETED,,0,2026-08-01T14:18:31+0800,2026-08-01T14:46:05+0800,1654.0,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,1619.8975476099877,bench_main_marker_plus_duration,64,0,1619.8975476099877,40.3918658202939,,4194304,65536,0.039508671455442484,2589.2402925038787,40.456879570373104,2629.6971720742517,,1022354.4435263402,950470.9995904996,1619639.9038552889,1619650.3429244931,785995.8297684161,774078.3500669932,1506265.7001026294,1560787.9135349835,231.04458822866482,223.80146996136563,418.57784627036636,439.7380822666596,231.0445591993489,56.0161369794514,56.77989884279668,58.514122637570836,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c64/cases/rdma_confirm_i65536_o1024_c64/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c64/cases/rdma_confirm_i65536_o1024_c64/rep1/bench.log,fixed_rdma_confirm_i65536_o1024_c64 +fixed_rdma_confirm_i65536_o1024_c64,fixed,rdma_confirm_i65536_o1024_c64,,rdma_confirm,2,65536,1024,64,64,0,COMPLETED,,0,2026-08-01T14:46:06+0800,2026-08-01T15:13:40+0800,1654.0,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,1620.2422464910196,bench_main_marker_plus_duration,64,0,1620.2422464910196,40.39704233770943,,4194304,65536,0.03950026617229964,2588.689443867829,40.44827256043483,2629.137716428264,,1022703.0410756742,950775.2832530241,1620010.8873196847,1620019.4079715456,786335.3449273454,774403.2157540205,1506724.735757988,1561196.831805848,231.05346642065376,223.83287831180922,418.5783493891426,439.7222371669007,231.0534379278651,56.055732507957146,56.75308680802118,58.11881933768746,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c64/cases/rdma_confirm_i65536_o1024_c64/rep2/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c64/cases/rdma_confirm_i65536_o1024_c64/rep2/bench.log,fixed_rdma_confirm_i65536_o1024_c64 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_rdma_summary.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_rdma_summary.csv new file mode 100644 index 0000000..2097456 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_rdma_summary.csv @@ -0,0 +1,25 @@ +phase2_bench_run,case_id,role,stage,repetition,status,started_at,ended_at,duration_s,window_source,node,hca,samples,xmit_bytes,rcv_bytes,xmit_gbps,rcv_gbps,port_xmit_wait_delta,port_xmit_discards_delta,port_rcv_errors_delta,req_transport_retries_exceeded_delta,req_rnr_retries_exceeded_delta,roce_adp_retrans_delta,roce_adp_retrans_to_delta,np_ecn_marked_roce_packets_delta,np_cnp_sent_delta,rp_cnp_handled_delta +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,1,COMPLETED,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,404.892570368,bench_main_marker_plus_duration,head,mlx5_0,371,3626828296476.0,3626837993952.0,71.66006119953522,71.66025280544177,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,1,COMPLETED,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,404.892570368,bench_main_marker_plus_duration,head,mlx5_3,371,3626914718800.0,3626921347028.0,71.66176876011448,71.66189972281393,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,1,COMPLETED,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,405.267634176,bench_main_marker_plus_duration,worker,mlx5_0,383,3626134447036.0,3626135801124.0,71.58004520955629,71.58007193930987,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,1,COMPLETED,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,405.267634176,bench_main_marker_plus_duration,worker,mlx5_3,383,3626207403788.0,3626212421504.0,71.58148537888337,71.58158442880648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,2,COMPLETED,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,405.0464832,bench_main_marker_plus_duration,head,mlx5_0,371,3625059758564.0,3625068124448.0,71.59790115790838,71.59806639097366,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,2,COMPLETED,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,405.0464832,bench_main_marker_plus_duration,head,mlx5_3,371,3625136689400.0,3625142126056.0,71.59942060496823,71.59952798337986,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,2,COMPLETED,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,404.277901824,bench_main_marker_plus_duration,worker,mlx5_0,382,3625042688652.0,3625045908572.0,71.73367967522776,71.73374339219049,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,2,COMPLETED,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,404.277901824,bench_main_marker_plus_duration,worker,mlx5_3,382,3625122942520.0,3625131203508.0,71.73526776832192,71.73543123979464,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,1,COMPLETED,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,126.9894016,bench_main_marker_plus_duration,head,mlx5_0,117,894188173332.0,894192013996.0,56.331515043976715,56.33175699575861,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,1,COMPLETED,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,126.9894016,bench_main_marker_plus_duration,head,mlx5_3,117,908654087048.0,908656485748.0,57.24282975425881,57.24298086608198,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,1,COMPLETED,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,127.337136896,bench_main_marker_plus_duration,worker,mlx5_0,121,908912097324.0,908914949500.0,57.10271925252004,57.102898441471176,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,1,COMPLETED,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,127.337136896,bench_main_marker_plus_duration,worker,mlx5_3,121,894353999288.0,894353927048.0,56.188101670195095,56.188097131691926,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,2,COMPLETED,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,124.79339136,bench_main_marker_plus_duration,head,mlx5_0,115,894195520320.0,894197433708.0,57.32326114869037,57.32338380826259,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,2,COMPLETED,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,124.79339136,bench_main_marker_plus_duration,head,mlx5_3,115,908661063200.0,908663318872.0,58.25058864399148,58.250733246007684,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,2,COMPLETED,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,124.054127104,bench_main_marker_plus_duration,worker,mlx5_0,118,908220840268.0,908222493124.0,58.56932688787363,58.5694334772174,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,2,COMPLETED,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,124.054127104,bench_main_marker_plus_duration,worker,mlx5_3,118,893986155676.0,893986644792.0,57.65136084035526,57.651392382457814,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,1,COMPLETED,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,1618.418803712,bench_main_marker_plus_duration,head,mlx5_0,1479,14610070965300.0,14610072489408.0,72.21898772698583,72.21899526079844,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,1,COMPLETED,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,1618.418803712,bench_main_marker_plus_duration,head,mlx5_3,1479,14610085802320.0,14610095102220.0,72.21906106780447,72.21910703810576,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,1,COMPLETED,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,1619.672280064,bench_main_marker_plus_duration,worker,mlx5_0,1527,14611153636236.0,14611146819600.0,72.16844452340027,72.1684108541891,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,1,COMPLETED,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,1619.672280064,bench_main_marker_plus_duration,worker,mlx5_3,1527,14611226802652.0,14611236350532.0,72.16880591214243,72.16885307170732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,2,COMPLETED,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,1619.210452736,bench_main_marker_plus_duration,head,mlx5_0,1480,14610569508844.0,14610573106108.0,72.18614224806339,72.18616002099213,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,2,COMPLETED,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,1619.210452736,bench_main_marker_plus_duration,head,mlx5_3,1480,14610597458160.0,14610605842440.0,72.18628033667787,72.18632176071877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,2,COMPLETED,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,1619.262776832,bench_main_marker_plus_duration,worker,mlx5_0,1527,14610266497856.0,14610260234736.0,72.18231262718308,72.18228168411397,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,2,COMPLETED,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,1619.262776832,bench_main_marker_plus_duration,worker,mlx5_3,1527,14610323772088.0,14610331114336.0,72.18259559166701,72.18263186618948,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_windows.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_windows.csv new file mode 100644 index 0000000..29fde54 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/case_windows.csv @@ -0,0 +1,7 @@ +phase2_bench_run,case_id,role,stage,repetition,status,started_at,ended_at,started_ns,ended_ns,duration_s,window_source,meta_path +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,1,COMPLETED,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,1785564254150768896,1785564660235236864,406.084467968,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c16/cases/rdma_confirm_i65536_o1024_c16/rep1/meta.json +fixed_rdma_confirm_i65536_o1024_c16,rdma_confirm_i65536_o1024_c16,,rdma_confirm,2,COMPLETED,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,1785564694981654016,1785565101074958080,406.093304064,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c16/cases/rdma_confirm_i65536_o1024_c16/rep2/meta.json +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,1,COMPLETED,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,1785563926377015040,1785564054095733760,127.71871872,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c4/cases/rdma_confirm_i65536_o1024_c4/rep1/meta.json +fixed_rdma_confirm_i65536_o1024_c4,rdma_confirm_i65536_o1024_c4,,rdma_confirm,2,COMPLETED,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,1785564088360896000,1785564213670408960,125.30951296,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c4/cases/rdma_confirm_i65536_o1024_c4/rep2/meta.json +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,1,COMPLETED,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,1785565142582675968,1785566762480224000,1619.897548032,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c64/cases/rdma_confirm_i65536_o1024_c64/rep1/meta.json +fixed_rdma_confirm_i65536_o1024_c64,rdma_confirm_i65536_o1024_c64,,rdma_confirm,2,COMPLETED,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,1785566797465815040,1785568417708060928,1620.242245888,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm/bench/fixed_rdma_confirm_i65536_o1024_c64/cases/rdma_confirm_i65536_o1024_c64/rep2/meta.json diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/collector_status.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/collector_status.csv new file mode 100644 index 0000000..293ab99 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/collector_status.csv @@ -0,0 +1,37 @@ +wall_time_ns,node,collector,status,pid +1785563885833567643,head,gpu_samples.csv,STARTED,2581028 +1785563885848767676,head,rdma.csv,STARTED,2581039 +1785563885862486752,head,dcgm_dmon.log,STARTED,2581053 +1785563885876598354,head,mpstat.log,STARTED,2581071 +1785563885891181421,head,pidstat.log,STARTED,2581090 +1785563885906513050,head,sar_net.log,STARTED,2581117 +1785563885921729008,head,perf_stat.log,STARTED,2581144 +1785563885938632713,head,docker_top.log,STARTED,2581179 +1785563885959612586,head,numa_samples.csv,STARTED,2581251 +1785563885980492840,worker,gpu_samples.csv,STARTED,2581294 +1785563885999012597,worker,rdma.csv,STARTED,2581356 +1785563886014677475,worker,dcgm_dmon.log,STARTED,2581408 +1785563886031239484,worker,mpstat.log,STARTED,2581455 +1785563886046314326,worker,pidstat.log,STARTED,2581464 +1785563886060934631,worker,sar_net.log,STARTED,2581478 +1785563886075086128,worker,perf_stat.log,STARTED,2581486 +1785563886090622653,worker,docker_top.log,STARTED,2581497 +1785563886106525841,worker,numa_samples.csv,STARTED,2581509 +1785568431501511663,head,gpu_samples.csv,STOPPED,2581028 +1785568431506303146,head,rdma.csv,STOPPED,2581039 +1785568431513026775,head,dcgm_dmon.log,STOPPED,2581053 +1785568431518181488,head,mpstat.log,STOPPED,2581071 +1785568431523384390,head,pidstat.log,STOPPED,2581090 +1785568431528223936,head,sar_net.log,STOPPED,2581117 +1785568431608439580,head,perf_stat.log,STOPPED,2581144 +1785568431612888149,head,docker_top.log,STOPPED,2581179 +1785568431617027570,head,numa_samples.csv,STOPPED,2581251 +1785568431620791766,worker,gpu_samples.csv,STOPPED,2581294 +1785568431624694474,worker,rdma.csv,STOPPED,2581356 +1785568431628587426,worker,dcgm_dmon.log,STOPPED,2581408 +1785568431632294659,worker,mpstat.log,STOPPED,2581455 +1785568431636153536,worker,pidstat.log,STOPPED,2581464 +1785568431639870020,worker,sar_net.log,STOPPED,2581478 +1785568431643739953,worker,perf_stat.log,STOPPED,2581486 +1785568431647558030,worker,docker_top.log,STOPPED,2581497 +1785568431651292267,worker,numa_samples.csv,STOPPED,2581509 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_nccl_transport.log b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_nccl_transport.log new file mode 100644 index 0000000..557e7f5 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_nccl_transport.log @@ -0,0 +1,1012 @@ +/usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). +/sgl-workspace/sglang/python/sglang/launch_server.py:56: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-01 05:53:04] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:04] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:04] Breakable CUDA graph is incompatible with DeepSeek-V4 (heavy capture-pool memory pressure); disabling prefill CUDA graph. +[2026-08-01 05:53:04] Use dsv4 attention backend for DeepseekV4ForCausalLM, setting page_size to 256. +[2026-08-01 05:53:04] Setting swa_full_tokens_ratio to 0.1 for DeepseekV4ForCausalLM. +[2026-08-01 05:53:04] Setting KV cache dtype to fp8_e4m3 for DeepseekV4ForCausalLM. +[2026-08-01 05:53:04] Use flashinfer_mxfp4 as MoE runner backend on SM120 for DeepseekV4 +[2026-08-01 05:53:07] server_args=ServerArgs(model_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=None, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.9, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=8192, enable_dynamic_chunking=False, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, retraction_policy='length', schedule_conservativeness=1.0, page_size=256, swa_full_tokens_ratio=0.1, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, enable_page_major_kv_layout=False, enable_unified_memory=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, nccl_port=None, dist_timeout=None, dist_init_addr='10.101.0.11:20002', nnodes=2, node_rank=0, tp_size=16, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=1, load_balance_method='round_robin', attn_cp_size=1, moe_dp_size=1, enable_prefill_cp=False, cp_strategy=None, enable_dsa_cache_layer_split=False, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_dp_attention=False, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=False, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=653015482, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, host='0.0.0.0', port=30002, fastapi_root_path='', smg_grpc_mode=False, grpc_mode=False, grpc_port=None, skip_server_warmup=False, warmups=None, enable_http2=False, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, api_key=None, admin_api_key=None, served_model_name='/data/hf_models/DeepSeek-V4-Pro', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, default_chat_template_kwargs=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser=None, tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, smg_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, attention_backend='dsv4', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='auto', bf16_gemm_backend='auto', dsa_prefill_backend=None, dsa_decode_backend=None, dsa_paged_mqa_logits_backend='auto', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, mamba_backend='triton', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=64, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], tc_compiler='eager', full_prefill_max_req=None), prefill=PhaseConfig(backend='disabled', max_bs=8192, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], tc_compiler='eager', full_prefill_max_req=None)), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=64, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, torchao_config='', speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_dspark_block_size=None, speculative_dspark_sps_table_path=None, speculative_dspark_confidence_sts_path=None, speculative_dspark_align_verify_tokens_to_graph_tier=False, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_window_size=None, speculative_moe_runner_backend='flashinfer_mxfp4', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=2, moe_a2a_backend='none', moe_runner_backend='flashinfer_mxfp4', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', fuseep_mode=2, deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_waterfill=False, ep_join_mode=None, ep_join_rank_offset=0, elastic_ep_initial_size=None, max_ep_size=None, elastic_ep_scale_timeout=600, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=False, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, enable_mamba_cache_stochastic_rounding=False, mamba_cache_philox_rounds=0, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', uses_mamba_radix_cache=False, mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, enable_mm_global_cache=False, disable_fast_image_processor=False, mm_feature_transport='cpu', keep_mm_feature_on_device=False, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', enable_lmcache=False, lmcache_config_file=None, enable_flexkv=False, flexkv_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, dllm_fdfo=True, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_attempts=0, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, forward_hooks=None, msprobe_dump_config=None) +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:07] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:07] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:10] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:10] No HuggingFace chat template found +[2026-08-01 05:53:10] No chat template found, defaulting to 'string' content format +[2026-08-01 05:53:21 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:21 TP0 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:21 TP2 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:21 TP2 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:22 TP1 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:22 TP1 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:22 TP5 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:22 TP5 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:22 TP4 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:22 TP4 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:22 TP3 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:22 TP3 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:22 TP6 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:22 TP6 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:22 TP7 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:22 TP7 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:23 TP0 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP2 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP1 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:24 TP0 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:24 TP2 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:24 TP2 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:24 TP1 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:24 TP1 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:24 TP0 EP0] Init torch distributed begin. +[2026-08-01 05:53:24 TP2 EP0] Init torch distributed begin. +[2026-08-01 05:53:24 TP1 EP0] Init torch distributed begin. +[2026-08-01 05:53:24 TP5 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP4 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP3 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP6 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:24 TP5 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:24 TP5 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:24 TP4 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:24 TP4 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:25 TP5 EP0] Init torch distributed begin. +[2026-08-01 05:53:25 TP7 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:25 TP4 EP0] Init torch distributed begin. +[2026-08-01 05:53:25 TP3 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:25 TP3 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:25 TP6 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:25 TP6 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:25 TP3 EP0] Init torch distributed begin. +[2026-08-01 05:53:25 TP6 EP0] Init torch distributed begin. +[2026-08-01 05:53:25 TP7 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:25 TP7 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:25 TP7 EP0] Init torch distributed begin. +[2026-08-01 05:53:32 TP0 EP0] sglang is using nccl==2.28.9 +gpu1:236:236 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:236:236 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:236:236 [0] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:240:240 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:240:240 [4] NCCL INFO cudaDriverVersion 13000 +gpu1:240:240 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:240:240 [4] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:240:240 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:236:236 [0] NCCL INFO cudaDriverVersion 13000 +gpu1:241:241 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:236:236 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:237:237 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:241:241 [5] NCCL INFO cudaDriverVersion 13000 +gpu1:237:237 [1] NCCL INFO cudaDriverVersion 13000 +gpu1:239:239 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:239:239 [3] NCCL INFO cudaDriverVersion 13000 +gpu1:243:243 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:242:242 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:243:243 [7] NCCL INFO cudaDriverVersion 13000 +gpu1:242:242 [6] NCCL INFO cudaDriverVersion 13000 +gpu1:238:238 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:238:238 [2] NCCL INFO cudaDriverVersion 13000 +gpu1:241:241 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:237:237 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:239:239 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:243:243 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:242:242 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:238:238 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:241:241 [5] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:241:241 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:237:237 [1] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:239:239 [3] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:237:237 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:239:239 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:242:242 [6] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:238:238 [2] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:243:243 [7] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:242:242 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:238:238 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:243:243 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:243:243 [7] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:243:243 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:243:243 [7] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:240:240 [4] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:242:242 [6] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:240:240 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:242:242 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:240:240 [4] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:242:242 [6] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:242:242 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:242:242 [6] NCCL INFO Initialized NET plugin IB +gpu1:242:242 [6] NCCL INFO Assigned NET plugin IB to comm +gpu1:242:242 [6] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:242:242 [6] NCCL INFO Using network IB +gpu1:242:242 [6] NCCL INFO DMA-BUF is available on GPU device 6 +gpu1:242:242 [6] NCCL INFO ncclCommInitRank comm 0x3a8d2010 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x2b119358eab7dcff - Init START +gpu1:243:243 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:243:243 [7] NCCL INFO Initialized NET plugin IB +gpu1:243:243 [7] NCCL INFO Assigned NET plugin IB to comm +gpu1:243:243 [7] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:243:243 [7] NCCL INFO Using network IB +gpu1:243:243 [7] NCCL INFO DMA-BUF is available on GPU device 7 +gpu1:243:243 [7] NCCL INFO ncclCommInitRank comm 0x28b2b750 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x2b119358eab7dcff - Init START +gpu1:243:243 [7] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:241:241 [5] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:240:240 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:240:240 [4] NCCL INFO Initialized NET plugin IB +gpu1:240:240 [4] NCCL INFO Assigned NET plugin IB to comm +gpu1:240:240 [4] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:240:240 [4] NCCL INFO Using network IB +gpu1:241:241 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:241:241 [5] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:239:239 [3] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:238:238 [2] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:239:239 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:239:239 [3] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:238:238 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:238:238 [2] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:240:240 [4] NCCL INFO DMA-BUF is available on GPU device 4 +gpu1:240:240 [4] NCCL INFO ncclCommInitRank comm 0x36d9f1d0 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x2b119358eab7dcff - Init START +gpu1:241:241 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:241:241 [5] NCCL INFO Initialized NET plugin IB +gpu1:241:241 [5] NCCL INFO Assigned NET plugin IB to comm +gpu1:241:241 [5] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:241:241 [5] NCCL INFO Using network IB +gpu1:239:239 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:239:239 [3] NCCL INFO Initialized NET plugin IB +gpu1:239:239 [3] NCCL INFO Assigned NET plugin IB to comm +gpu1:239:239 [3] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:239:239 [3] NCCL INFO Using network IB +gpu1:238:238 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:238:238 [2] NCCL INFO Initialized NET plugin IB +gpu1:238:238 [2] NCCL INFO Assigned NET plugin IB to comm +gpu1:238:238 [2] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:238:238 [2] NCCL INFO Using network IB +gpu1:237:237 [1] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:237:237 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:237:237 [1] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:241:241 [5] NCCL INFO DMA-BUF is available on GPU device 5 +gpu1:241:241 [5] NCCL INFO ncclCommInitRank comm 0x444cf2b0 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x2b119358eab7dcff - Init START +gpu1:241:241 [5] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:239:239 [3] NCCL INFO DMA-BUF is available on GPU device 3 +gpu1:239:239 [3] NCCL INFO ncclCommInitRank comm 0x44fa5530 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x2b119358eab7dcff - Init START +gpu1:238:238 [2] NCCL INFO DMA-BUF is available on GPU device 2 +gpu1:238:238 [2] NCCL INFO ncclCommInitRank comm 0x3fb0ede0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x2b119358eab7dcff - Init START +gpu1:242:242 [6] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:240:240 [4] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:239:239 [3] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:237:237 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:237:237 [1] NCCL INFO Initialized NET plugin IB +gpu1:237:237 [1] NCCL INFO Assigned NET plugin IB to comm +gpu1:237:237 [1] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:237:237 [1] NCCL INFO Using network IB +gpu1:236:236 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:236:236 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:236:236 [0] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:237:237 [1] NCCL INFO DMA-BUF is available on GPU device 1 +gpu1:237:237 [1] NCCL INFO ncclCommInitRank comm 0x301409e0 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x2b119358eab7dcff - Init START +gpu1:238:238 [2] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:236:236 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:236:236 [0] NCCL INFO Initialized NET plugin IB +gpu1:236:236 [0] NCCL INFO Assigned NET plugin IB to comm +gpu1:236:236 [0] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:236:236 [0] NCCL INFO Using network IB +gpu1:236:236 [0] NCCL INFO DMA-BUF is available on GPU device 0 +gpu1:236:236 [0] NCCL INFO ncclCommInitRank comm 0x2b6502a0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x2b119358eab7dcff - Init START +gpu1:236:236 [0] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:237:237 [1] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:236:236 [0] NCCL INFO Bootstrap timings total 0.001337 (create 0.000021, send 0.000083, recv 0.000347, ring 0.000587, delay 0.000001) +gpu1:236:236 [0] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:237:237 [1] NCCL INFO Bootstrap timings total 0.029616 (create 0.000036, send 0.000140, recv 0.000158, ring 0.000553, delay 0.000001) +gpu1:237:237 [1] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:239:239 [3] NCCL INFO Bootstrap timings total 0.055744 (create 0.000029, send 0.000099, recv 0.000197, ring 0.054966, delay 0.000001) +gpu1:238:238 [2] NCCL INFO Bootstrap timings total 0.055600 (create 0.000029, send 0.000102, recv 0.000248, ring 0.029003, delay 0.000001) +gpu1:239:239 [3] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:238:238 [2] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:241:241 [5] NCCL INFO Bootstrap timings total 0.059111 (create 0.000032, send 0.000110, recv 0.000280, ring 0.058374, delay 0.000001) +gpu1:241:241 [5] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:243:243 [7] NCCL INFO Bootstrap timings total 0.079050 (create 0.000051, send 0.000181, recv 0.000306, ring 0.077971, delay 0.000002) +gpu1:240:240 [4] NCCL INFO Bootstrap timings total 0.067663 (create 0.000055, send 0.000239, recv 0.008583, ring 0.054965, delay 0.000002) +gpu1:243:243 [7] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:240:240 [4] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:242:242 [6] NCCL INFO Bootstrap timings total 0.089982 (create 0.000044, send 0.000155, recv 0.011074, ring 0.055592, delay 0.000001) +gpu1:242:242 [6] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:239:239 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:239:239 [3] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:239:239 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:238:238 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:238:238 [2] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:238:238 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:241:241 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:241:241 [5] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:241:241 [5] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:237:237 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:237:237 [1] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:237:237 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:240:240 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:240:240 [4] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:240:240 [4] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:236:236 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:236:236 [0] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:243:243 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:243:243 [7] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:243:243 [7] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:236:236 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:242:242 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:242:242 [6] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:242:242 [6] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:241:241 [5] NCCL INFO comm 0x444cf2b0 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:240:240 [4] NCCL INFO comm 0x36d9f1d0 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:241:241 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->7 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->7 +gpu1:241:241 [5] NCCL INFO P2P Chunksize set to 131072 +gpu1:238:238 [2] NCCL INFO comm 0x3fb0ede0 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:239:239 [3] NCCL INFO comm 0x44fa5530 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu1:240:240 [4] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->6 [2] 5/8/0->4->12 [3] -1/-1/-1->4->6 +gpu1:237:237 [1] NCCL INFO comm 0x301409e0 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:240:240 [4] NCCL INFO P2P Chunksize set to 131072 +gpu1:236:236 [0] NCCL INFO comm 0x2b6502a0 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:242:242 [6] NCCL INFO comm 0x3a8d2010 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:243:243 [7] NCCL INFO comm 0x28b2b750 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu1:236:236 [0] NCCL INFO Channel 00/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:239:239 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 1/11/-1->3->-1 [2] -1/-1/-1->3->2 [3] 1/-1/-1->3->7 +gpu1:237:237 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->3 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->3 +gpu1:236:236 [0] NCCL INFO Channel 01/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:243:243 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 5/-1/-1->7->11 [2] -1/-1/-1->7->6 [3] 5/11/3->7->15 +gpu1:237:237 [1] NCCL INFO P2P Chunksize set to 131072 +gpu1:243:243 [7] NCCL INFO P2P Chunksize set to 131072 +gpu1:238:238 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 0/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 0/-1/-1->2->1 +gpu1:239:239 [3] NCCL INFO P2P Chunksize set to 131072 +gpu1:236:236 [0] NCCL INFO Channel 02/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 03/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:238:238 [2] NCCL INFO P2P Chunksize set to 131072 +gpu1:242:242 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 4/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 4/-1/-1->6->5 +gpu1:236:236 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->2 [2] 1/-1/-1->0->4 [3] -1/-1/-1->0->2 +gpu1:242:242 [6] NCCL INFO P2P Chunksize set to 131072 +gpu1:236:236 [0] NCCL INFO P2P Chunksize set to 131072 +gpu1:241:241 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:241:241 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:243:243 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:243:243 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:240:240 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:240:240 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:1584 [0] NCCL INFO [Proxy Service] Device 5 CPU core 100 +gpu1:241:1585 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 124 +gpu1:242:242 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:242:242 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:243:1586 [0] NCCL INFO [Proxy Service] Device 7 CPU core 35 +gpu1:237:237 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:240:1587 [0] NCCL INFO [Proxy Service] Device 4 CPU core 37 +gpu1:237:237 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:243:1588 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 33 +gpu1:236:236 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:236:236 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:240:1589 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 110 +gpu1:238:238 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:239:239 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:238:238 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:239:239 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:242:1590 [0] NCCL INFO [Proxy Service] Device 6 CPU core 111 +gpu1:242:1591 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 48 +gpu1:237:1592 [0] NCCL INFO [Proxy Service] Device 1 CPU core 70 +gpu1:236:1593 [0] NCCL INFO [Proxy Service] Device 0 CPU core 4 +gpu1:237:1594 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 71 +gpu1:236:1595 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 9 +gpu1:238:1597 [0] NCCL INFO [Proxy Service] Device 2 CPU core 10 +gpu1:239:1599 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 76 +gpu1:238:1598 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 75 +gpu1:239:1596 [0] NCCL INFO [Proxy Service] Device 3 CPU core 10 +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:240:1600 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 113 +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:1601 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 114 +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:236:1602 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 8 +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:239:1603 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 14 +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:241:241 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:239:239 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:242:242 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:238:238 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:237:237 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Connected all trees +gpu1:240:240 [4] NCCL INFO Connected all trees +gpu1:237:237 [1] NCCL INFO Connected all trees +gpu1:241:241 [5] NCCL INFO Connected all trees +gpu1:243:243 [7] NCCL INFO Connected all trees +gpu1:239:239 [3] NCCL INFO Connected all trees +gpu1:238:238 [2] NCCL INFO Connected all trees +gpu1:242:242 [6] NCCL INFO Connected all trees +gpu1:243:243 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:240:240 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:243:243 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:243:243 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:240:240 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:240:240 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:239:239 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:236:236 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:236:236 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:239:239 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:239:239 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO CC Off, workFifoBytes 1048576 +gpu1:241:1604 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 101 +gpu1:242:1605 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 115 +gpu1:241:241 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:241:241 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:241:241 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:242:242 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:242:242 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:242:242 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:243:243 [7] NCCL INFO ncclCommInitRank comm 0x28b2b750 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:243:243 [7] NCCL INFO Init timings - ncclCommInitRank: rank 7 nranks 16 total 0.64 (kernels 0.26, alloc 0.04, bootstrap 0.08, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.17, rest 0.05) +gpu1:241:241 [5] NCCL INFO ncclCommInitRank comm 0x444cf2b0 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:241:241 [5] NCCL INFO Init timings - ncclCommInitRank: rank 5 nranks 16 total 0.64 (kernels 0.30, alloc 0.02, bootstrap 0.06, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.22, rest 0.00) +gpu1:242:242 [6] NCCL INFO ncclCommInitRank comm 0x3a8d2010 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:240:240 [4] NCCL INFO ncclCommInitRank comm 0x36d9f1d0 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:242:242 [6] NCCL INFO Init timings - ncclCommInitRank: rank 6 nranks 16 total 0.64 (kernels 0.27, alloc 0.02, bootstrap 0.09, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.22, rest 0.00) +gpu1:240:240 [4] NCCL INFO Init timings - ncclCommInitRank: rank 4 nranks 16 total 0.67 (kernels 0.30, alloc 0.04, bootstrap 0.07, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.17, rest 0.05) +gpu1:238:1606 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 79 +gpu1:237:1607 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 16 +gpu1:237:237 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:237:237 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:237:237 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:238:238 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:238:238 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:238:238 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:238:238 [2] NCCL INFO ncclCommInitRank comm 0x3fb0ede0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:238:238 [2] NCCL INFO Init timings - ncclCommInitRank: rank 2 nranks 16 total 0.65 (kernels 0.30, alloc 0.02, bootstrap 0.06, allgathers 0.00, topo 0.03, graphs 0.01, connections 0.22, rest 0.00) +gpu1:237:237 [1] NCCL INFO ncclCommInitRank comm 0x301409e0 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:237:237 [1] NCCL INFO Init timings - ncclCommInitRank: rank 1 nranks 16 total 0.65 (kernels 0.31, alloc 0.03, bootstrap 0.03, allgathers 0.01, topo 0.03, graphs 0.01, connections 0.22, rest 0.00) +gpu1:239:239 [3] NCCL INFO ncclCommInitRank comm 0x44fa5530 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:239:239 [3] NCCL INFO Init timings - ncclCommInitRank: rank 3 nranks 16 total 0.65 (kernels 0.30, alloc 0.02, bootstrap 0.06, allgathers 0.00, topo 0.03, graphs 0.01, connections 0.17, rest 0.05) +gpu1:236:236 [0] NCCL INFO ncclCommInitRank comm 0x2b6502a0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu1:236:236 [0] NCCL INFO Init timings - ncclCommInitRank: rank 0 nranks 16 total 0.65 (kernels 0.35, alloc 0.03, bootstrap 0.00, allgathers 0.00, topo 0.04, graphs 0.01, connections 0.17, rest 0.05) +[2026-08-01 05:53:32 TP0 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP1 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP2 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP3 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP4 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP5 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP6 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP7 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:33 TP0 EP0] Init torch distributed ends. elapsed=8.43 s, mem usage=0.42 GB +[2026-08-01 05:53:33 TP7 EP0] Init torch distributed ends. elapsed=7.52 s, mem usage=0.48 GB +[2026-08-01 05:53:33 TP6 EP0] Init torch distributed ends. elapsed=7.74 s, mem usage=0.39 GB +[2026-08-01 05:53:33 TP5 EP0] Init torch distributed ends. elapsed=7.94 s, mem usage=0.41 GB +[2026-08-01 05:53:33 TP4 EP0] Init torch distributed ends. elapsed=7.90 s, mem usage=0.46 GB +[2026-08-01 05:53:33 TP3 EP0] Init torch distributed ends. elapsed=7.75 s, mem usage=0.44 GB +[2026-08-01 05:53:33 TP2 EP0] Init torch distributed ends. elapsed=8.41 s, mem usage=0.39 GB +[2026-08-01 05:53:33 TP1 EP0] Init torch distributed ends. elapsed=8.39 s, mem usage=0.41 GB +[2026-08-01 05:53:36 TP4 EP0] Load weight begin. avail mem=82.04 GB +[2026-08-01 05:53:36 TP7 EP0] Load weight begin. avail mem=82.03 GB +[2026-08-01 05:53:36 TP0 EP0] Load weight begin. avail mem=82.08 GB +[2026-08-01 05:53:36 TP0 EP0] Detected fp8 checkpoint. +[2026-08-01 05:53:36 TP3 EP0] Load weight begin. avail mem=82.07 GB +[2026-08-01 05:53:36 TP0 EP0] Config does not support fused shared expert(s). Shared experts fusion optimization is disabled. +[2026-08-01 05:53:36 TP2 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:53:36 TP1 EP0] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:53:36 TP6 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:53:36 TP5 EP0] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:53:36 TP4 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP7 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP3 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP2 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP1 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP6 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP5 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:40 TP1 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP2 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP0 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP3 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP4 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP7 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP6 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP5 EP0] multimem all-gather disabled because the TP group spans across nodes. + Multi-thread loading shards: 0% Completed | 0/64 [00:002->1 [1] 0/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 0/-1/-1->2->1 +gpu1:238:238 [2] NCCL INFO P2P Chunksize set to 131072 +gpu1:238:238 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:240:240 [4] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->6 [2] 5/8/0->4->12 [3] -1/-1/-1->4->6 +gpu1:241:241 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->7 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->7 +gpu1:240:240 [4] NCCL INFO P2P Chunksize set to 131072 +gpu1:241:241 [5] NCCL INFO P2P Chunksize set to 131072 +gpu1:242:242 [6] NCCL INFO comm 0x5e4f4a90 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:239:239 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 1/11/-1->3->-1 [2] -1/-1/-1->3->2 [3] 1/-1/-1->3->7 +gpu1:240:240 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:241 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:239:239 [3] NCCL INFO P2P Chunksize set to 131072 +gpu1:242:242 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 4/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 4/-1/-1->6->5 +gpu1:242:242 [6] NCCL INFO P2P Chunksize set to 131072 +gpu1:239:239 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:243:243 [7] NCCL INFO comm 0x53716a40 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu1:242:242 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:243:243 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 5/-1/-1->7->11 [2] -1/-1/-1->7->6 [3] 5/11/3->7->15 +gpu1:243:243 [7] NCCL INFO P2P Chunksize set to 131072 +gpu1:243:243 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:238:2377 [0] NCCL INFO [Proxy Service] Device 2 CPU core 83 +gpu1:242:2378 [0] NCCL INFO [Proxy Service] Device 6 CPU core 40 +gpu1:238:2380 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 18 +gpu1:240:2379 [0] NCCL INFO [Proxy Service] Device 4 CPU core 105 +gpu1:242:2383 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 42 +gpu1:239:2381 [0] NCCL INFO [Proxy Service] Device 3 CPU core 2 +gpu1:240:2384 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 107 +gpu1:243:2382 [0] NCCL INFO [Proxy Service] Device 7 CPU core 106 +gpu1:241:2385 [0] NCCL INFO [Proxy Service] Device 5 CPU core 108 +gpu1:239:2387 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 3 +gpu1:243:2386 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 45 +gpu1:241:2388 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 46 +gpu1:237:237 [1] NCCL INFO comm 0x514b5460 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:236:236 [0] NCCL INFO comm 0x63214490 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:237:237 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->3 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->3 +gpu1:237:237 [1] NCCL INFO P2P Chunksize set to 131072 +gpu1:237:237 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:236:236 [0] NCCL INFO Channel 00/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 01/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 02/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 03/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->2 [2] 1/-1/-1->0->4 [3] -1/-1/-1->0->2 +gpu1:236:236 [0] NCCL INFO P2P Chunksize set to 131072 +gpu1:236:236 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:237:2389 [0] NCCL INFO [Proxy Service] Device 1 CPU core 15 +gpu1:237:2391 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 86 +gpu1:236:2390 [0] NCCL INFO [Proxy Service] Device 0 CPU core 4 +gpu1:236:2392 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 5 +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:240:2393 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 111 +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:236:2394 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 6 +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:243:2395 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 49 +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:239:2396 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 71 +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:236:236 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:242:242 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:239:239 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:237:237 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:238:238 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:240:240 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [send] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Connected all trees +gpu1:237:237 [1] NCCL INFO Connected all trees +gpu1:239:239 [3] NCCL INFO Connected all trees +gpu1:238:238 [2] NCCL INFO Connected all trees +gpu1:240:240 [4] NCCL INFO Connected all trees +gpu1:241:241 [5] NCCL INFO Connected all trees +gpu1:243:243 [7] NCCL INFO Connected all trees +gpu1:242:242 [6] NCCL INFO Connected all trees +gpu1:243:243 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:243:243 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:240:240 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:240:240 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:239:239 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:239:239 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:236:236 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO CC Off, workFifoBytes 1048576 +gpu1:241:2397 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 114 +gpu1:241:241 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:241:241 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:242:2398 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 41 +gpu1:242:242 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:242:242 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:243:243 [7] NCCL INFO ncclCommInitRankConfig comm 0x53716a40 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:241:241 [5] NCCL INFO ncclCommInitRankConfig comm 0x6b1e9550 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:243:243 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 16 total 0.31 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.06, graphs 0.01, connections 0.17, rest 0.05) +gpu1:241:241 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.01, topo 0.06, graphs 0.00, connections 0.22, rest 0.00) +gpu1:240:240 [4] NCCL INFO ncclCommInitRankConfig comm 0x73011510 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:242:242 [6] NCCL INFO ncclCommInitRankConfig comm 0x5e4f4a90 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:240:240 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.01, topo 0.07, graphs 0.01, connections 0.17, rest 0.05) +gpu1:242:242 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.01, topo 0.07, graphs 0.00, connections 0.22, rest 0.00) +gpu1:237:2399 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 17 +gpu1:238:2400 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 87 +gpu1:237:237 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:237:237 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:238:238 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:238:238 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:238:238 [2] NCCL INFO ncclCommInitRankConfig comm 0x5d984f60 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:238:238 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.01, topo 0.07, graphs 0.00, connections 0.23, rest 0.00) +gpu1:236:236 [0] NCCL INFO ncclCommInitRankConfig comm 0x63214490 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:236:236 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.01, topo 0.06, graphs 0.01, connections 0.17, rest 0.06) +gpu1:237:237 [1] NCCL INFO ncclCommInitRankConfig comm 0x514b5460 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:237:237 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 16 total 0.31 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.01, topo 0.06, graphs 0.00, connections 0.23, rest 0.00) +gpu1:239:239 [3] NCCL INFO ncclCommInitRankConfig comm 0x6fa816f0 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x37d27f86da993abb - Init COMPLETE +gpu1:239:239 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.00, topo 0.07, graphs 0.01, connections 0.17, rest 0.06) +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-08-01 05:56:23 TP7 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP7 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP7 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.69 GB +[2026-08-01 05:56:23 TP0 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP0 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP0 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.80 GB +[2026-08-01 05:56:23 TP2 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP2 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP1 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP1 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP3 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP3 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP6 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP6 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP5 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP5 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP4 EP0] FlashInfer autotune completed. +[2026-08-01 05:56:23 TP4 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:23 TP2 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:56:23 TP1 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:56:23 TP3 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.77 GB +[2026-08-01 05:56:23 TP6 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:56:23 TP5 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:56:23 TP4 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.72 GB + 0%| | 0/12 [00:00, ) +[2026-08-01 05:57:52 TP2 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP3 EP0] Capture target decode CUDA graph end. elapsed=89.50 s, mem usage=1.37 GB, avail mem=5.40 GB. +[2026-08-01 05:57:52 TP1 EP0] Capture target decode CUDA graph end. elapsed=89.50 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:57:52 TP6 EP0] Capture target decode CUDA graph end. elapsed=89.50 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:57:52 TP3 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP3 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP1 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP1 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP6 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP6 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP7 EP0] Capture target decode CUDA graph end. elapsed=89.56 s, mem usage=1.37 GB, avail mem=5.32 GB. +[2026-08-01 05:57:52 TP5 EP0] Capture target decode CUDA graph end. elapsed=89.51 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:57:52 TP7 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP7 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP5 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP5 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP4 EP0] Capture target decode CUDA graph end. elapsed=89.53 s, mem usage=1.37 GB, avail mem=5.35 GB. +[2026-08-01 05:57:52 TP4 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP4 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP0 EP0] Capture target decode CUDA graph end. elapsed=89.57 s, mem usage=1.37 GB, avail mem=5.43 GB. +[2026-08-01 05:57:52 TP0 EP0] max_total_num_tokens=1270016, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=256, context_len=1048576, available_gpu_mem=5.43 GB +[2026-08-01 05:57:52 TP0 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP0 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:53] INFO: Started server process [1] +[2026-08-01 05:57:53] INFO: Waiting for application startup. +[2026-08-01 05:57:53] Using default chat sampling params from model generation config: {'temperature': 1.0, 'top_p': 1.0} +[2026-08-01 05:57:53] INFO: Application startup complete. +[2026-08-01 05:57:53] INFO: Uvicorn running on http://0.0.0.0:30002 (Press CTRL+C to quit) +[2026-08-01 05:57:54] INFO: 127.0.0.1:33108 - "GET /model_info HTTP/1.1" 200 OK +[2026-08-01 05:57:54] INFO: 10.101.0.11:47524 - "GET /health HTTP/1.1" 503 Service Unavailable +[2026-08-01 05:57:58 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 45.81 +[2026-08-01 05:57:58] INFO: 127.0.0.1:33120 - "POST /generate HTTP/1.1" 200 OK +[2026-08-01 05:57:58] The server is fired up and ready to roll! +[2026-08-01 05:58:00 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 100.09 +[2026-08-01 05:58:00] INFO: 10.101.0.11:48966 - "GET /health HTTP/1.1" 200 OK diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_server_cmd.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_server_cmd.txt new file mode 100644 index 0000000..ff6de9b --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/head_server_cmd.txt @@ -0,0 +1 @@ +docker run -d --name dsv4pro_pro6000d_2node_sglang_tp16_quick_map_head --gpus all --network host --ipc host --shm-size 20g --ulimit memlock=-1 --ulimit stack=67108864 -v /data/hf_models/DeepSeek-V4-Pro:/data/hf_models/DeepSeek-V4-Pro:ro -v /data/hzy/sglang_cache/dsv4_pro_tp16:/root/.cache -e CUDA_VISIBLE_DEVICES=0\,1\,2\,3\,4\,5\,6\,7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -e NCCL_SOCKET_IFNAME=eth0 -e NCCL_IB_HCA==mlx5_0:1\,mlx5_3:1 -e NCCL_CROSS_NIC=1 -e NCCL_DEBUG=INFO -e SGLANG_SHARED_EXPERT_TP1=1 --device /dev/infiniband/rdma_cm --device /dev/infiniband/uverbs0 --device /dev/infiniband/uverbs3 --entrypoint python3 lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45 -m sglang.launch_server --model-path /data/hf_models/DeepSeek-V4-Pro --tp-size 16 --ep-size 2 --nnodes 2 --node-rank 0 --dist-init-addr 10.101.0.11:20002 --trust-remote-code --host 0.0.0.0 --port 30002 --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_nccl_transport.log b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_nccl_transport.log new file mode 100644 index 0000000..58ffb22 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_nccl_transport.log @@ -0,0 +1,918 @@ +/usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). +/sgl-workspace/sglang/python/sglang/launch_server.py:56: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-01 05:52:58] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:52:58] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:52:58] Breakable CUDA graph is incompatible with DeepSeek-V4 (heavy capture-pool memory pressure); disabling prefill CUDA graph. +[2026-08-01 05:52:58] Use dsv4 attention backend for DeepseekV4ForCausalLM, setting page_size to 256. +[2026-08-01 05:52:58] Setting swa_full_tokens_ratio to 0.1 for DeepseekV4ForCausalLM. +[2026-08-01 05:52:58] Setting KV cache dtype to fp8_e4m3 for DeepseekV4ForCausalLM. +[2026-08-01 05:52:58] Use flashinfer_mxfp4 as MoE runner backend on SM120 for DeepseekV4 +[2026-08-01 05:53:01] server_args=ServerArgs(model_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=None, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.9, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=8192, enable_dynamic_chunking=False, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, retraction_policy='length', schedule_conservativeness=1.0, page_size=256, swa_full_tokens_ratio=0.1, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, enable_page_major_kv_layout=False, enable_unified_memory=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, nccl_port=None, dist_timeout=None, dist_init_addr='10.101.0.11:20002', nnodes=2, node_rank=1, tp_size=16, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=1, load_balance_method='round_robin', attn_cp_size=1, moe_dp_size=1, enable_prefill_cp=False, cp_strategy=None, enable_dsa_cache_layer_split=False, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_dp_attention=False, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=False, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=805559529, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, host='0.0.0.0', port=30002, fastapi_root_path='', smg_grpc_mode=False, grpc_mode=False, grpc_port=None, skip_server_warmup=False, warmups=None, enable_http2=False, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, api_key=None, admin_api_key=None, served_model_name='/data/hf_models/DeepSeek-V4-Pro', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, default_chat_template_kwargs=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser=None, tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, smg_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, attention_backend='dsv4', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='auto', bf16_gemm_backend='auto', dsa_prefill_backend=None, dsa_decode_backend=None, dsa_paged_mqa_logits_backend='auto', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, mamba_backend='triton', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=64, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], tc_compiler='eager', full_prefill_max_req=None), prefill=PhaseConfig(backend='disabled', max_bs=8192, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], tc_compiler='eager', full_prefill_max_req=None)), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=64, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, torchao_config='', speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_dspark_block_size=None, speculative_dspark_sps_table_path=None, speculative_dspark_confidence_sts_path=None, speculative_dspark_align_verify_tokens_to_graph_tier=False, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_window_size=None, speculative_moe_runner_backend='flashinfer_mxfp4', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=2, moe_a2a_backend='none', moe_runner_backend='flashinfer_mxfp4', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', fuseep_mode=2, deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_waterfill=False, ep_join_mode=None, ep_join_rank_offset=0, elastic_ep_initial_size=None, max_ep_size=None, elastic_ep_scale_timeout=600, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=False, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, enable_mamba_cache_stochastic_rounding=False, mamba_cache_philox_rounds=0, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', uses_mamba_radix_cache=False, mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, enable_mm_global_cache=False, disable_fast_image_processor=False, mm_feature_transport='cpu', keep_mm_feature_on_device=False, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', enable_lmcache=False, lmcache_config_file=None, enable_flexkv=False, flexkv_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, dllm_fdfo=True, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_attempts=0, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, forward_hooks=None, msprobe_dump_config=None) +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:53:14 TP8 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:14 TP8 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:14 TP12 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:14 TP12 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:14 TP15 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:14 TP15 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:15 TP11 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:15 TP11 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:15 TP13 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:15 TP13 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:15 TP10 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:15 TP10 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:15 TP14 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:15 TP14 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:15 TP9 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:15 TP9 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:16 TP8 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:16 TP15 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:16 TP8 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:16 TP8 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:16 TP12 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:17 TP15 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:17 TP15 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:17 TP8 EP1] Init torch distributed begin. +[2026-08-01 05:53:17 TP12 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:17 TP12 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:17 TP15 EP1] Init torch distributed begin. +[2026-08-01 05:53:17 TP11 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:17 TP10 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:17 TP13 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:17 TP12 EP1] Init torch distributed begin. +[2026-08-01 05:53:17 TP9 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:17 TP10 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:17 TP10 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:17 TP11 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:17 TP11 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:17 TP13 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:17 TP13 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:17 TP14 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:53:17 TP10 EP1] Init torch distributed begin. +[2026-08-01 05:53:17 TP11 EP1] Init torch distributed begin. +[2026-08-01 05:53:17 TP13 EP1] Init torch distributed begin. +[2026-08-01 05:53:17 TP9 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:17 TP9 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:18 TP14 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:53:18 TP14 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:53:18 TP9 EP1] Init torch distributed begin. +[2026-08-01 05:53:18 TP14 EP1] Init torch distributed begin. +gpu3:236:236 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:236:236 [1] NCCL INFO cudaDriverVersion 13000 +gpu3:236:236 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:236:236 [1] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:236:236 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:235:235 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:235:235 [0] NCCL INFO cudaDriverVersion 13000 +gpu3:239:239 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:239:239 [4] NCCL INFO cudaDriverVersion 13000 +gpu3:235:235 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:239:239 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:235:235 [0] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:235:235 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:239:239 [4] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:239:239 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:238:238 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:238:238 [3] NCCL INFO cudaDriverVersion 13000 +gpu3:238:238 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:238:238 [3] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:238:238 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:240:240 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:240:240 [5] NCCL INFO cudaDriverVersion 13000 +gpu3:240:240 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:242:242 [7] NCCL INFO cudaDriverVersion 13000 +gpu3:240:240 [5] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:240:240 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:241:241 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:241:241 [6] NCCL INFO cudaDriverVersion 13000 +gpu3:237:237 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:237:237 [2] NCCL INFO cudaDriverVersion 13000 +gpu3:242:242 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:241:241 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:237:237 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [7] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:242:242 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:241:241 [6] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:241:241 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:237:237 [2] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:237:237 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:240:240 [5] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:240:240 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:240:240 [5] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:238:238 [3] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:238:238 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:238:238 [3] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:238:238 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:238:238 [3] NCCL INFO Initialized NET plugin IB +gpu3:238:238 [3] NCCL INFO Assigned NET plugin IB to comm +gpu3:238:238 [3] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:238:238 [3] NCCL INFO Using network IB +gpu3:238:238 [3] NCCL INFO DMA-BUF is available on GPU device 3 +gpu3:238:238 [3] NCCL INFO ncclCommInitRank comm 0x432bbca0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x2b119358eab7dcff - Init START +gpu3:240:240 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:240:240 [5] NCCL INFO Initialized NET plugin IB +gpu3:240:240 [5] NCCL INFO Assigned NET plugin IB to comm +gpu3:240:240 [5] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:240:240 [5] NCCL INFO Using network IB +gpu3:240:240 [5] NCCL INFO DMA-BUF is available on GPU device 5 +gpu3:240:240 [5] NCCL INFO ncclCommInitRank comm 0x13a383e0 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x2b119358eab7dcff - Init START +gpu3:236:236 [1] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:236:236 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:236:236 [1] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:235:235 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:235:235 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:235:235 [0] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:236:236 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:236:236 [1] NCCL INFO Initialized NET plugin IB +gpu3:236:236 [1] NCCL INFO Assigned NET plugin IB to comm +gpu3:236:236 [1] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:236:236 [1] NCCL INFO Using network IB +gpu3:242:242 [7] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:242:242 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [7] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:235:235 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:235:235 [0] NCCL INFO Initialized NET plugin IB +gpu3:235:235 [0] NCCL INFO Assigned NET plugin IB to comm +gpu3:235:235 [0] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:235:235 [0] NCCL INFO Using network IB +gpu3:236:236 [1] NCCL INFO DMA-BUF is available on GPU device 1 +gpu3:236:236 [1] NCCL INFO ncclCommInitRank comm 0x46b43d40 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x2b119358eab7dcff - Init START +gpu3:235:235 [0] NCCL INFO DMA-BUF is available on GPU device 0 +gpu3:235:235 [0] NCCL INFO ncclCommInitRank comm 0x36e47db0 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x2b119358eab7dcff - Init START +gpu3:242:242 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:242:242 [7] NCCL INFO Initialized NET plugin IB +gpu3:242:242 [7] NCCL INFO Assigned NET plugin IB to comm +gpu3:242:242 [7] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:242:242 [7] NCCL INFO Using network IB +gpu3:241:241 [6] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:241:241 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [7] NCCL INFO DMA-BUF is available on GPU device 7 +gpu3:242:242 [7] NCCL INFO ncclCommInitRank comm 0x3c31ce70 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x2b119358eab7dcff - Init START +gpu3:241:241 [6] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:239:239 [4] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:239:239 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:235:235 [0] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:239:239 [4] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:237:237 [2] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:237:237 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:237:237 [2] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:241:241 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:241:241 [6] NCCL INFO Initialized NET plugin IB +gpu3:241:241 [6] NCCL INFO Assigned NET plugin IB to comm +gpu3:241:241 [6] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:241:241 [6] NCCL INFO Using network IB +gpu3:239:239 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:239:239 [4] NCCL INFO Initialized NET plugin IB +gpu3:239:239 [4] NCCL INFO Assigned NET plugin IB to comm +gpu3:239:239 [4] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:239:239 [4] NCCL INFO Using network IB +gpu3:237:237 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:237:237 [2] NCCL INFO Initialized NET plugin IB +gpu3:237:237 [2] NCCL INFO Assigned NET plugin IB to comm +gpu3:237:237 [2] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:237:237 [2] NCCL INFO Using network IB +gpu3:241:241 [6] NCCL INFO DMA-BUF is available on GPU device 6 +gpu3:241:241 [6] NCCL INFO ncclCommInitRank comm 0x4d5de030 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x2b119358eab7dcff - Init START +gpu3:241:241 [6] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:239:239 [4] NCCL INFO DMA-BUF is available on GPU device 4 +gpu3:239:239 [4] NCCL INFO ncclCommInitRank comm 0x33a4c690 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x2b119358eab7dcff - Init START +gpu3:239:239 [4] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:240:240 [5] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:237:237 [2] NCCL INFO DMA-BUF is available on GPU device 2 +gpu3:237:237 [2] NCCL INFO ncclCommInitRank comm 0x4edb3050 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x2b119358eab7dcff - Init START +gpu3:236:236 [1] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:238:238 [3] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:237:237 [2] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:242:242 [7] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:242:242 [7] NCCL INFO Bootstrap timings total 0.099960 (create 0.000024, send 0.000126, recv 0.098935, ring 0.000559, delay 0.000001) +gpu3:241:241 [6] NCCL INFO Bootstrap timings total 0.062953 (create 0.000027, send 0.000191, recv 0.000493, ring 0.061936, delay 0.000001) +gpu3:242:242 [7] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:241:241 [6] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:240:240 [5] NCCL INFO Bootstrap timings total 0.121542 (create 0.000025, send 0.000155, recv 0.058990, ring 0.056753, delay 0.000001) +gpu3:240:240 [5] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:239:239 [4] NCCL INFO Bootstrap timings total 0.058725 (create 0.000037, send 0.000168, recv 0.000510, ring 0.057539, delay 0.000001) +gpu3:239:239 [4] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:238:238 [3] NCCL INFO Bootstrap timings total 0.136361 (create 0.000032, send 0.000190, recv 0.078009, ring 0.055211, delay 0.000001) +gpu3:238:238 [3] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:237:237 [2] NCCL INFO Bootstrap timings total 0.056292 (create 0.000032, send 0.000135, recv 0.000569, ring 0.055191, delay 0.000001) +gpu3:236:236 [1] NCCL INFO Bootstrap timings total 0.109326 (create 0.000024, send 0.000176, recv 0.053454, ring 0.055388, delay 0.000001) +gpu3:237:237 [2] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:236:236 [1] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:235:235 [0] NCCL INFO Bootstrap timings total 0.105555 (create 0.000024, send 0.000122, recv 0.000361, ring 0.078050, delay 0.000001) +gpu3:235:235 [0] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:237:237 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:237:237 [2] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:237:237 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:238:238 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:238:238 [3] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:241:241 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:238:238 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:241:241 [6] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:241:241 [6] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:239:239 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:239:239 [4] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:239:239 [4] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:240:240 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:240:240 [5] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:240:240 [5] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:236:236 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:235:235 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:236:236 [1] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:235:235 [0] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:236:236 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:235:235 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:242:242 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:242:242 [7] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:242:242 [7] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:240:240 [5] NCCL INFO comm 0x13a383e0 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:240:240 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->15 [2] 14/-1/-1->13->12 [3] 14/-1/-1->13->15 +gpu3:240:240 [5] NCCL INFO P2P Chunksize set to 131072 +gpu3:239:239 [4] NCCL INFO comm 0x33a4c690 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:241:241 [6] NCCL INFO comm 0x4d5de030 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:238:238 [3] NCCL INFO comm 0x432bbca0 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:239:239 [4] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->14 [2] 13/4/-1->12->-1 [3] -1/-1/-1->12->14 +gpu3:239:239 [4] NCCL INFO P2P Chunksize set to 131072 +gpu3:242:242 [7] NCCL INFO comm 0x3c31ce70 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:241:241 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 12/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 12/-1/-1->14->13 +gpu3:241:241 [6] NCCL INFO P2P Chunksize set to 131072 +gpu3:242:242 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 13/-1/-1->15->11 [2] -1/-1/-1->15->14 [3] 13/7/-1->15->-1 +gpu3:238:238 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 9/7/15->11->3 [2] -1/-1/-1->11->10 [3] 9/-1/-1->11->7 +gpu3:242:242 [7] NCCL INFO P2P Chunksize set to 131072 +gpu3:238:238 [3] NCCL INFO P2P Chunksize set to 131072 +gpu3:237:237 [2] NCCL INFO comm 0x4edb3050 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:236:236 [1] NCCL INFO comm 0x46b43d40 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:235:235 [0] NCCL INFO comm 0x36e47db0 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:237:237 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 8/-1/-1->10->9 [2] 11/-1/-1->10->9 [3] 8/-1/-1->10->9 +gpu3:237:237 [2] NCCL INFO P2P Chunksize set to 131072 +gpu3:236:236 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/-1/-1->9->11 [2] 10/-1/-1->9->8 [3] 10/-1/-1->9->11 +gpu3:236:236 [1] NCCL INFO P2P Chunksize set to 131072 +gpu3:235:235 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->10 [2] 9/-1/-1->8->4 [3] -1/-1/-1->8->10 +gpu3:235:235 [0] NCCL INFO P2P Chunksize set to 131072 +gpu3:240:240 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:241:241 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:240:240 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:241:241 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:242:242 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:242:242 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:239:239 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:239:239 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:238:238 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:240:1435 [0] NCCL INFO [Proxy Service] Device 5 CPU core 102 +gpu3:238:238 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:241:1436 [0] NCCL INFO [Proxy Service] Device 6 CPU core 59 +gpu3:237:237 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:236:236 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:237:237 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:239:1438 [0] NCCL INFO [Proxy Service] Device 4 CPU core 37 +gpu3:236:236 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:242:1437 [0] NCCL INFO [Proxy Service] Device 7 CPU core 63 +gpu3:240:1440 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 106 +gpu3:239:1442 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 115 +gpu3:242:1441 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 34 +gpu3:235:235 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:241:1439 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 39 +gpu3:235:235 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:238:1443 [0] NCCL INFO [Proxy Service] Device 3 CPU core 3 +gpu3:238:1444 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 4 +gpu3:236:1446 [0] NCCL INFO [Proxy Service] Device 1 CPU core 70 +gpu3:235:1449 [0] NCCL INFO [Proxy Service] Device 0 CPU core 8 +gpu3:237:1445 [0] NCCL INFO [Proxy Service] Device 2 CPU core 69 +gpu3:236:1448 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 71 +gpu3:237:1447 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 73 +gpu3:235:1450 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 10 +gpu3:240:240 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 01/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 01/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 03/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 02/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 03/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 03/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:239:1451 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 107 +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 01/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 03/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:235:1452 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 11 +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:242:1453 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 45 +gpu3:235:235 [0] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 01/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:238:1454 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 70 +gpu3:238:238 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:240:240 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:238:238 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:239:239 [4] NCCL INFO Channel 01/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 01/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:242:242 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:241:241 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:236:236 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:236:236 [1] NCCL INFO Channel 01/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:235:235 [0] NCCL INFO Channel 01/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 03/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 03/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 01/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:237:237 [2] NCCL INFO Channel 01/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [send] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:237:237 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 03/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [send] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [send] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 00/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 02/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 00/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 02/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 02/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 00/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 03/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 02/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Connected all trees +gpu3:240:240 [5] NCCL INFO Connected all trees +gpu3:235:235 [0] NCCL INFO Connected all trees +gpu3:242:242 [7] NCCL INFO Connected all trees +gpu3:241:241 [6] NCCL INFO Connected all trees +gpu3:236:236 [1] NCCL INFO Connected all trees +gpu3:238:238 [3] NCCL INFO Connected all trees +gpu3:237:237 [2] NCCL INFO Connected all trees +gpu3:242:242 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:242:242 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:242:242 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:235:235 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:235:235 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:235:235 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:239:239 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:239:239 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:239:239 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:238:238 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:238:238 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:238:238 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:240:1455 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 40 +gpu3:241:1456 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 61 +gpu3:240:240 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:240:240 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:240:240 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:241:241 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:241:241 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:241:241 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:241:241 [6] NCCL INFO ncclCommInitRank comm 0x4d5de030 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:241:241 [6] NCCL INFO Init timings - ncclCommInitRank: rank 14 nranks 16 total 0.65 (kernels 0.28, alloc 0.04, bootstrap 0.06, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.22, rest 0.00) +gpu3:236:1457 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 76 +gpu3:236:236 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:236:236 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:236:236 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:240:240 [5] NCCL INFO ncclCommInitRank comm 0x13a383e0 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:242:242 [7] NCCL INFO ncclCommInitRank comm 0x3c31ce70 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:240:240 [5] NCCL INFO Init timings - ncclCommInitRank: rank 13 nranks 16 total 0.65 (kernels 0.23, alloc 0.03, bootstrap 0.12, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.22, rest 0.00) +gpu3:242:242 [7] NCCL INFO Init timings - ncclCommInitRank: rank 15 nranks 16 total 0.65 (kernels 0.27, alloc 0.01, bootstrap 0.10, allgathers 0.02, topo 0.03, graphs 0.00, connections 0.17, rest 0.05) +gpu3:239:239 [4] NCCL INFO ncclCommInitRank comm 0x33a4c690 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:239:239 [4] NCCL INFO Init timings - ncclCommInitRank: rank 12 nranks 16 total 0.67 (kernels 0.33, alloc 0.02, bootstrap 0.06, allgathers 0.02, topo 0.02, graphs 0.01, connections 0.17, rest 0.05) +gpu3:237:1458 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 77 +gpu3:237:237 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:237:237 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:237:237 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:237:237 [2] NCCL INFO ncclCommInitRank comm 0x4edb3050 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:236:236 [1] NCCL INFO ncclCommInitRank comm 0x46b43d40 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:238:238 [3] NCCL INFO ncclCommInitRank comm 0x432bbca0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:237:237 [2] NCCL INFO Init timings - ncclCommInitRank: rank 10 nranks 16 total 0.65 (kernels 0.31, alloc 0.02, bootstrap 0.06, allgathers 0.02, topo 0.02, graphs 0.01, connections 0.22, rest 0.00) +gpu3:236:236 [1] NCCL INFO Init timings - ncclCommInitRank: rank 9 nranks 16 total 0.68 (kernels 0.29, alloc 0.01, bootstrap 0.11, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.22, rest 0.00) +gpu3:235:235 [0] NCCL INFO ncclCommInitRank comm 0x36e47db0 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x2b119358eab7dcff - Init COMPLETE +gpu3:238:238 [3] NCCL INFO Init timings - ncclCommInitRank: rank 11 nranks 16 total 0.67 (kernels 0.25, alloc 0.01, bootstrap 0.14, allgathers 0.02, topo 0.02, graphs 0.01, connections 0.17, rest 0.05) +gpu3:235:235 [0] NCCL INFO Init timings - ncclCommInitRank: rank 8 nranks 16 total 0.68 (kernels 0.29, alloc 0.01, bootstrap 0.11, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.17, rest 0.06) +[2026-08-01 05:53:32 TP15 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP14 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP13 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP12 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP11 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP10 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP9 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:32 TP8 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:53:33 TP15 EP1] Init torch distributed ends. elapsed=15.72 s, mem usage=0.44 GB +[2026-08-01 05:53:33 TP14 EP1] Init torch distributed ends. elapsed=14.87 s, mem usage=0.39 GB +[2026-08-01 05:53:33 TP13 EP1] Init torch distributed ends. elapsed=15.21 s, mem usage=0.41 GB +[2026-08-01 05:53:33 TP11 EP1] Init torch distributed ends. elapsed=15.22 s, mem usage=0.48 GB +[2026-08-01 05:53:33 TP12 EP1] Init torch distributed ends. elapsed=15.64 s, mem usage=0.42 GB +[2026-08-01 05:53:33 TP10 EP1] Init torch distributed ends. elapsed=15.22 s, mem usage=0.39 GB +[2026-08-01 05:53:33 TP9 EP1] Init torch distributed ends. elapsed=14.96 s, mem usage=0.41 GB +[2026-08-01 05:53:33 TP8 EP1] Init torch distributed ends. elapsed=15.83 s, mem usage=0.46 GB +[2026-08-01 05:53:36 TP14 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:53:36 TP11 EP1] Load weight begin. avail mem=82.03 GB +[2026-08-01 05:53:36 TP10 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:53:36 TP8 EP1] Load weight begin. avail mem=82.04 GB +[2026-08-01 05:53:36 TP9 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:53:36 TP12 EP1] Load weight begin. avail mem=82.08 GB +[2026-08-01 05:53:36 TP13 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:53:36 TP15 EP1] Load weight begin. avail mem=82.07 GB +[2026-08-01 05:53:36 TP14 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP10 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP8 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP11 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP12 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP9 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP15 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:36 TP13 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:53:40 TP15 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP14 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP13 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP11 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP12 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP10 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP9 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:40 TP8 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:53:58 TP11 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:58 TP8 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:58 TP14 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:58 TP10 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:59 TP9 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:59 TP13 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:59 TP15 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:53:59 TP12 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:55:20 TP8 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:20 TP8 EP1] Load weight end. elapsed=104.81 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.17 GB, mem usage=56.87 GB. +[2026-08-01 05:55:20 TP10 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:20 TP10 EP1] Load weight end. elapsed=104.91 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.24 GB, mem usage=56.87 GB. +[2026-08-01 05:55:21 TP12 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:21 TP12 EP1] Load weight end. elapsed=105.60 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.21 GB, mem usage=56.87 GB. +[2026-08-01 05:55:21 TP11 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:21 TP11 EP1] Load weight end. elapsed=105.77 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.16 GB, mem usage=56.87 GB. +[2026-08-01 05:55:22 TP13 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:22 TP13 EP1] Load weight end. elapsed=106.17 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.23 GB, mem usage=56.87 GB. +[2026-08-01 05:55:22 TP14 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:22 TP14 EP1] Load weight end. elapsed=106.88 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.24 GB, mem usage=56.87 GB. +[2026-08-01 05:55:23 TP9 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:23 TP9 EP1] Load weight end. elapsed=107.09 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.23 GB, mem usage=56.87 GB. +[2026-08-01 05:55:23 TP15 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:55:23 TP15 EP1] Load weight end. elapsed=107.83 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.19 GB, mem usage=56.87 GB. +[2026-08-01 05:55:52 TP13 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP9 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP14 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP10 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP12 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP8 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP11 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP15 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:55:52 TP15 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP15 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP14 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP14 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP10 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP12 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP10 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP13 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP8 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP12 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP8 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP13 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP11 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP11 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP9 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:55:52 TP9 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:55:52 TP11 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP15 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP10 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP9 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP12 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP13 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP8 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:52 TP14 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:55:54 TP11 EP1] Memory pool end. avail mem=7.09 GB +[2026-08-01 05:55:54 TP9 EP1] Memory pool end. avail mem=7.16 GB +[2026-08-01 05:55:54 TP10 EP1] Memory pool end. avail mem=7.18 GB +[2026-08-01 05:55:54 TP13 EP1] Memory pool end. avail mem=7.16 GB +[2026-08-01 05:55:54 TP15 EP1] Memory pool end. avail mem=7.13 GB +[2026-08-01 05:55:54 TP14 EP1] Memory pool end. avail mem=7.18 GB +[2026-08-01 05:55:54 TP12 EP1] Memory pool end. avail mem=7.14 GB +[2026-08-01 05:55:54 TP8 EP1] Memory pool end. avail mem=7.10 GB +[2026-08-01 05:55:54 TP11 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP11 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp11_pp0_dp0.json +[2026-08-01 05:55:54 TP9 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP13 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP9 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp9_pp0_dp0.json +[2026-08-01 05:55:54 TP13 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp13_pp0_dp0.json +[2026-08-01 05:55:54 TP15 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP15 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp15_pp0_dp0.json +[2026-08-01 05:55:54 TP10 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP10 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp10_pp0_dp0.json +[2026-08-01 05:55:54 TP12 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP12 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp12_pp0_dp0.json +[2026-08-01 05:55:54 TP8 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP8 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp8_pp0_dp0.json +[2026-08-01 05:55:54 TP14 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:55:54 TP14 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp14_pp0_dp0.json +gpu3:241:241 [6] NCCL INFO Comm config Blocking set to 1 +gpu3:235:235 [0] NCCL INFO Comm config Blocking set to 1 +gpu3:242:242 [7] NCCL INFO Comm config Blocking set to 1 +gpu3:236:236 [1] NCCL INFO Comm config Blocking set to 1 +gpu3:239:239 [4] NCCL INFO Comm config Blocking set to 1 +gpu3:238:238 [3] NCCL INFO Comm config Blocking set to 1 +gpu3:240:240 [5] NCCL INFO Comm config Blocking set to 1 +gpu3:237:237 [2] NCCL INFO Comm config Blocking set to 1 +gpu3:235:235 [0] NCCL INFO Initialized NET plugin IB +gpu3:237:237 [2] NCCL INFO Initialized NET plugin IB +gpu3:235:235 [0] NCCL INFO Assigned NET plugin IB to comm +gpu3:235:235 [0] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:237:237 [2] NCCL INFO Assigned NET plugin IB to comm +gpu3:235:235 [0] NCCL INFO Using network IB +gpu3:237:237 [2] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:237:237 [2] NCCL INFO Using network IB +gpu3:235:235 [0] NCCL INFO DMA-BUF is available on GPU device 0 +gpu3:237:237 [2] NCCL INFO DMA-BUF is available on GPU device 2 +gpu3:241:241 [6] NCCL INFO Initialized NET plugin IB +gpu3:241:241 [6] NCCL INFO Assigned NET plugin IB to comm +gpu3:241:241 [6] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:235:235 [0] NCCL INFO ncclCommInitRankConfig comm 0x63acf010 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x37d27f86da993abb - Init START +gpu3:241:241 [6] NCCL INFO Using network IB +gpu3:241:241 [6] NCCL INFO DMA-BUF is available on GPU device 6 +gpu3:237:237 [2] NCCL INFO ncclCommInitRankConfig comm 0x7b50a560 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x37d27f86da993abb - Init START +gpu3:239:239 [4] NCCL INFO Initialized NET plugin IB +gpu3:239:239 [4] NCCL INFO Assigned NET plugin IB to comm +gpu3:239:239 [4] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:239:239 [4] NCCL INFO Using network IB +gpu3:241:241 [6] NCCL INFO ncclCommInitRankConfig comm 0x792a2470 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x37d27f86da993abb - Init START +gpu3:239:239 [4] NCCL INFO DMA-BUF is available on GPU device 4 +gpu3:239:239 [4] NCCL INFO ncclCommInitRankConfig comm 0x584f9ce0 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x37d27f86da993abb - Init START +gpu3:238:238 [3] NCCL INFO Initialized NET plugin IB +gpu3:238:238 [3] NCCL INFO Assigned NET plugin IB to comm +gpu3:238:238 [3] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:238:238 [3] NCCL INFO Using network IB +gpu3:238:238 [3] NCCL INFO DMA-BUF is available on GPU device 3 +gpu3:238:238 [3] NCCL INFO ncclCommInitRankConfig comm 0x615928b0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x37d27f86da993abb - Init START +gpu3:236:236 [1] NCCL INFO Initialized NET plugin IB +gpu3:236:236 [1] NCCL INFO Assigned NET plugin IB to comm +gpu3:236:236 [1] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:236:236 [1] NCCL INFO Using network IB +gpu3:236:236 [1] NCCL INFO DMA-BUF is available on GPU device 1 +gpu3:242:242 [7] NCCL INFO Initialized NET plugin IB +gpu3:242:242 [7] NCCL INFO Assigned NET plugin IB to comm +gpu3:242:242 [7] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:242:242 [7] NCCL INFO Using network IB +gpu3:236:236 [1] NCCL INFO ncclCommInitRankConfig comm 0x7471f640 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x37d27f86da993abb - Init START +gpu3:242:242 [7] NCCL INFO DMA-BUF is available on GPU device 7 +gpu3:242:242 [7] NCCL INFO ncclCommInitRankConfig comm 0x5a887d80 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x37d27f86da993abb - Init START +gpu3:240:240 [5] NCCL INFO Initialized NET plugin IB +gpu3:240:240 [5] NCCL INFO Assigned NET plugin IB to comm +gpu3:240:240 [5] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:240:240 [5] NCCL INFO Using network IB +gpu3:240:240 [5] NCCL INFO DMA-BUF is available on GPU device 5 +gpu3:240:240 [5] NCCL INFO ncclCommInitRankConfig comm 0x489e8680 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x37d27f86da993abb - Init START +gpu3:236:236 [1] NCCL INFO Bootstrap timings total 0.008882 (create 0.000058, send 0.000217, recv 0.000943, ring 0.007285, delay 0.000000) +gpu3:235:235 [0] NCCL INFO Bootstrap timings total 0.011116 (create 0.000058, send 0.000857, recv 0.002383, ring 0.002103, delay 0.000000) +gpu3:238:238 [3] NCCL INFO Bootstrap timings total 0.009379 (create 0.000055, send 0.000226, recv 0.000808, ring 0.007983, delay 0.000000) +gpu3:237:237 [2] NCCL INFO Bootstrap timings total 0.010755 (create 0.000065, send 0.000295, recv 0.001933, ring 0.007369, delay 0.000000) +gpu3:240:240 [5] NCCL INFO Bootstrap timings total 0.008146 (create 0.000057, send 0.000291, recv 0.000821, ring 0.006596, delay 0.000000) +gpu3:239:239 [4] NCCL INFO Bootstrap timings total 0.010482 (create 0.000053, send 0.000287, recv 0.002980, ring 0.006795, delay 0.000000) +gpu3:241:241 [6] NCCL INFO Bootstrap timings total 0.010728 (create 0.000051, send 0.000239, recv 0.002867, ring 0.006715, delay 0.000000) +gpu3:242:242 [7] NCCL INFO Bootstrap timings total 0.008852 (create 0.000057, send 0.000284, recv 0.001104, ring 0.007050, delay 0.000000) +gpu3:242:242 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:238:238 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:240:240 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:235:235 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:239:239 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:236:236 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:241:241 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:237:237 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:240:240 [5] NCCL INFO comm 0x489e8680 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:239:239 [4] NCCL INFO comm 0x584f9ce0 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:240:240 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->15 [2] 14/-1/-1->13->12 [3] 14/-1/-1->13->15 +gpu3:240:240 [5] NCCL INFO P2P Chunksize set to 131072 +gpu3:238:238 [3] NCCL INFO comm 0x615928b0 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:240:240 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:239:239 [4] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->14 [2] 13/4/-1->12->-1 [3] -1/-1/-1->12->14 +gpu3:239:239 [4] NCCL INFO P2P Chunksize set to 131072 +gpu3:238:238 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 9/7/15->11->3 [2] -1/-1/-1->11->10 [3] 9/-1/-1->11->7 +gpu3:239:239 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:238:238 [3] NCCL INFO P2P Chunksize set to 131072 +gpu3:238:238 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:236:236 [1] NCCL INFO comm 0x7471f640 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:237:237 [2] NCCL INFO comm 0x7b50a560 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:235:235 [0] NCCL INFO comm 0x63acf010 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:236:236 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/-1/-1->9->11 [2] 10/-1/-1->9->8 [3] 10/-1/-1->9->11 +gpu3:237:237 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 8/-1/-1->10->9 [2] 11/-1/-1->10->9 [3] 8/-1/-1->10->9 +gpu3:236:236 [1] NCCL INFO P2P Chunksize set to 131072 +gpu3:237:237 [2] NCCL INFO P2P Chunksize set to 131072 +gpu3:236:236 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:237:237 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:235:235 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->10 [2] 9/-1/-1->8->4 [3] -1/-1/-1->8->10 +gpu3:235:235 [0] NCCL INFO P2P Chunksize set to 131072 +gpu3:235:235 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:240:2227 [0] NCCL INFO [Proxy Service] Device 5 CPU core 98 +gpu3:239:2228 [0] NCCL INFO [Proxy Service] Device 4 CPU core 60 +gpu3:238:2229 [0] NCCL INFO [Proxy Service] Device 3 CPU core 91 +gpu3:239:2230 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 62 +gpu3:240:2232 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 99 +gpu3:238:2231 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 26 +gpu3:236:2234 [0] NCCL INFO [Proxy Service] Device 1 CPU core 1 +gpu3:236:2237 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 67 +gpu3:237:2235 [0] NCCL INFO [Proxy Service] Device 2 CPU core 2 +gpu3:235:2236 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 93 +gpu3:237:2238 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 4 +gpu3:235:2233 [0] NCCL INFO [Proxy Service] Device 0 CPU core 92 +gpu3:241:241 [6] NCCL INFO comm 0x792a2470 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:241:241 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 12/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 12/-1/-1->14->13 +gpu3:241:241 [6] NCCL INFO P2P Chunksize set to 131072 +gpu3:241:241 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:242:242 [7] NCCL INFO comm 0x5a887d80 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:242:242 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 13/-1/-1->15->11 [2] -1/-1/-1->15->14 [3] 13/7/-1->15->-1 +gpu3:242:242 [7] NCCL INFO P2P Chunksize set to 131072 +gpu3:242:242 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:241:2239 [0] NCCL INFO [Proxy Service] Device 6 CPU core 58 +gpu3:242:2240 [0] NCCL INFO [Proxy Service] Device 7 CPU core 36 +gpu3:241:2241 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 62 +gpu3:242:2242 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 37 +gpu3:240:240 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 01/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 03/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 01/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 02/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 03/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 03/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:2243 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 31 +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:2245 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 102 +gpu3:235:2244 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 5 +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 01/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 01/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 03/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:242:2246 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 104 +gpu3:242:242 [7] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:239:239 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:240:240 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:238:238 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:239:239 [4] NCCL INFO Channel 01/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 01/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:237:237 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:236:236 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:242:242 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:235:235 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:236:236 [1] NCCL INFO Channel 01/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 01/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 03/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 03/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 01/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:241:241 [6] NCCL INFO Channel 03/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 01/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:237:237 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [send] via NET/IB/1/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [send] via NET/IB/1/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 01/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 01/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:235:235 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:235:235 [0] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [7] NCCL INFO Channel 03/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu3:238:238 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 00/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:238:238 [3] NCCL INFO Channel 02/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:242:242 [7] NCCL INFO Channel 02/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 00/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:240:240 [5] NCCL INFO Channel 02/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 00/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 02/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:236:236 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [6] NCCL INFO Channel 03/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:237:237 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:239:239 [4] NCCL INFO Connected all trees +gpu3:240:240 [5] NCCL INFO Connected all trees +gpu3:242:242 [7] NCCL INFO Connected all trees +gpu3:241:241 [6] NCCL INFO Connected all trees +gpu3:235:235 [0] NCCL INFO Connected all trees +gpu3:236:236 [1] NCCL INFO Connected all trees +gpu3:238:238 [3] NCCL INFO Connected all trees +gpu3:237:237 [2] NCCL INFO Connected all trees +gpu3:238:238 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:238:238 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:235:235 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:235:235 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:242:242 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:242:242 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:239:239 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:239:239 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:240:2247 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 105 +gpu3:240:240 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:240:240 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:241:2248 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 42 +gpu3:241:241 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:241:241 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:236:2250 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 7 +gpu3:237:2249 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 70 +gpu3:241:241 [6] NCCL INFO ncclCommInitRankConfig comm 0x792a2470 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:239:239 [4] NCCL INFO ncclCommInitRankConfig comm 0x584f9ce0 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:241:241 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 14 nranks 16 total 0.31 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.05, graphs 0.00, connections 0.22, rest 0.00) +gpu3:239:239 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 12 nranks 16 total 0.31 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.04, graphs 0.01, connections 0.17, rest 0.05) +gpu3:237:237 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:237:237 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:236:236 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:236:236 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:240:240 [5] NCCL INFO ncclCommInitRankConfig comm 0x489e8680 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:242:242 [7] NCCL INFO ncclCommInitRankConfig comm 0x5a887d80 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:242:242 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 15 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.04, graphs 0.01, connections 0.17, rest 0.05) +gpu3:240:240 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 13 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.04, graphs 0.01, connections 0.19, rest 0.03) +gpu3:237:237 [2] NCCL INFO ncclCommInitRankConfig comm 0x7b50a560 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:235:235 [0] NCCL INFO ncclCommInitRankConfig comm 0x63acf010 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:237:237 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 10 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.05, graphs 0.01, connections 0.22, rest 0.00) +gpu3:235:235 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 8 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.04, graphs 0.01, connections 0.17, rest 0.05) +gpu3:236:236 [1] NCCL INFO ncclCommInitRankConfig comm 0x7471f640 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:236:236 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 9 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.05, graphs 0.01, connections 0.22, rest 0.00) +gpu3:238:238 [3] NCCL INFO ncclCommInitRankConfig comm 0x615928b0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x37d27f86da993abb - Init COMPLETE +gpu3:238:238 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 11 nranks 16 total 0.32 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.03, topo 0.04, graphs 0.01, connections 0.17, rest 0.05) +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-08-01 05:56:22 TP14 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP14 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP8 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP8 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP14 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:56:22 TP8 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.72 GB +[2026-08-01 05:56:22 TP10 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP10 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP13 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP13 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP11 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP11 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP10 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:56:22 TP13 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:56:22 TP15 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP9 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP15 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP9 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP11 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.69 GB +[2026-08-01 05:56:22 TP15 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.77 GB +[2026-08-01 05:56:22 TP12 EP1] FlashInfer autotune completed. +[2026-08-01 05:56:22 TP12 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:56:22 TP9 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:56:22 TP12 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.80 GB +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +[2026-08-01 05:57:52 TP13 EP1] Capture target decode CUDA graph end. elapsed=89.68 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:57:52 TP13 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP13 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP12 EP1] Capture target decode CUDA graph end. elapsed=89.68 s, mem usage=1.37 GB, avail mem=5.43 GB. +[2026-08-01 05:57:52 TP12 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP12 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP8 EP1] Capture target decode CUDA graph end. elapsed=89.71 s, mem usage=1.37 GB, avail mem=5.35 GB. +[2026-08-01 05:57:52 TP8 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP8 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP11 EP1] Capture target decode CUDA graph end. elapsed=89.69 s, mem usage=1.37 GB, avail mem=5.32 GB. +[2026-08-01 05:57:52 TP9 EP1] Capture target decode CUDA graph end. elapsed=89.69 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:57:52 TP11 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP11 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP9 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP15 EP1] Capture target decode CUDA graph end. elapsed=89.69 s, mem usage=1.37 GB, avail mem=5.40 GB. +[2026-08-01 05:57:52 TP9 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP15 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP15 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP14 EP1] Capture target decode CUDA graph end. elapsed=89.73 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:57:52 TP14 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP14 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:52 TP10 EP1] Capture target decode CUDA graph end. elapsed=89.96 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:57:52 TP10 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:57:52 TP10 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:57:53] Dummy health check server started in background thread at 0.0.0.0:30002 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_server_cmd.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_server_cmd.txt new file mode 100644 index 0000000..636e480 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/confirm/service/worker_server_cmd.txt @@ -0,0 +1 @@ +docker run -d --name dsv4pro_pro6000d_2node_sglang_tp16_quick_map_worker --gpus all --network host --ipc host --shm-size 20g --ulimit memlock=-1 --ulimit stack=67108864 -v /data/hf_models/DeepSeek-V4-Pro:/data/hf_models/DeepSeek-V4-Pro:ro -v /data/hzy/sglang_cache/dsv4_pro_tp16:/root/.cache -e CUDA_VISIBLE_DEVICES=0\,1\,2\,3\,4\,5\,6\,7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -e NCCL_SOCKET_IFNAME=eth0 -e NCCL_IB_HCA==mlx5_0:1\,mlx5_3:1 -e NCCL_CROSS_NIC=1 -e NCCL_DEBUG=INFO -e SGLANG_SHARED_EXPERT_TP1=1 --device /dev/infiniband/rdma_cm --device /dev/infiniband/uverbs0 --device /dev/infiniband/uverbs3 --entrypoint python3 lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45 -m sglang.launch_server --model-path /data/hf_models/DeepSeek-V4-Pro --tp-size 16 --ep-size 2 --nnodes 2 --node-rank 1 --dist-init-addr 10.101.0.11:20002 --trust-remote-code --host 0.0.0.0 --port 30002 --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_case_metrics.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_case_metrics.csv new file mode 100644 index 0000000..5b17995 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_case_metrics.csv @@ -0,0 +1,12 @@ +stage,case_id,repetition,status,isl,osl,concurrency,completed,duration_s,input_tps,output_tps,total_tps,ttft_p95_ms,tpot_p95_ms,gpu_util_mean_pct,rail_xmit_avg_gbps,rail_xmit_p95_gbps,rail_xmit_max_gbps,dual_rail_xmit_avg_gbps,rail_imbalance_pct,xmit_bytes_per_input_token_per_rail,xmit_bytes_per_total_token_per_rail,rdma_error_delta,measurement_started_at,measurement_ended_at,source_result_dir +scout,rdma_scout_i65536_o1_c1,1,COMPLETED,65536,1,1,1,24.1862671909621,2709.6368150803128,0.04134577659729481,2709.67816085691,24167.605170048773,0.0,93.74897875816994,70.9712746848062,85.5972372945909,86.80566056858429,141.9425493696124,0.5207119247408836,3137732.762954712,3137684.8856828967,0.0,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout +scout,rdma_scout_i65536_o1_c4,1,COMPLETED,65536,1,4,4,89.45394851703895,2930.491100122512,0.044715745546302976,2930.535815868058,86526.69315790116,0.0,97.4494891826923,78.27464182068744,86.81919268342193,88.85894624954848,156.5492836413749,0.2703571902588796,3298012.1388320923,3297961.815928407,0.0,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout +scout,rdma_scout_i65536_o1_c16,1,COMPLETED,65536,1,16,16,351.42641380499117,2983.771164628114,0.045528734811830354,2983.816693362926,336717.4506017618,0.0,99.33137833282255,79.89685137869246,86.416508483831,88.20167661610235,159.79370275738495,0.1149304623247205,3335858.008945465,3335807.108568442,0.0,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout +scout,rdma_scout_i65536_o1_c32,1,COMPLETED,65536,1,32,32,702.8175486620166,2983.9209393567885,0.04553102019282209,2983.9664703769813,671139.2772838357,0.0,99.49393723047987,79.92852806348517,86.22587499587476,88.21644913628985,159.85705612697035,0.10091204027725086,3343929.3806581497,3343878.357123648,0.0,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout +scout,rdma_scout_i65536_o1_c64,1,COMPLETED,65536,1,64,64,1402.17612864502,2991.2818470623424,0.04564333873080967,2991.327490401073,1337131.1103800863,0.0,99.63928846991146,79.9743866369171,85.93278751515446,88.32212536165416,159.9487732738342,0.03496630506355109,3339597.901974201,3339546.9445318105,0.0,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout +confirm,rdma_confirm_i65536_o1024_c4,1,COMPLETED,65536,1024,4,4,127.71871886099689,2052.5104098899187,32.07047515452998,2084.5808850444487,87520.75181677355,96.14249014501952,97.83383458646617,56.71629143023766,86.06750356786138,88.99885476939444,113.43258286047534,1.8596562953363254,3439052.922241211,3386144.415745192,0.0,2026-08-01T13:58:46.377015+08:00,2026-08-01T14:00:54.095734+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm +confirm,rdma_confirm_i65536_o1024_c4,2,COMPLETED,65536,1024,4,4,125.30951252899831,2091.9720674784076,32.68706355435012,2124.6591310327576,85207.63279019155,95.11951526871552,98.63248207885304,57.94863438022769,86.4894823881113,88.5335715781298,115.89726876045536,2.1502935358359734,3438056.544746399,3385163.3671349157,0.0,2026-08-01T14:01:28.360896+08:00,2026-08-01T14:03:33.670409+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm +confirm,rdma_confirm_i65536_o1024_c16,1,COMPLETED,65536,1024,16,16,406.0844679859583,2582.1622905219265,40.3462857894051,2622.5085763113316,336778.7128902273,356.41907673483706,99.38883591065292,71.62084013702234,86.26345231858909,88.9399300384103,143.24168027404468,0.11410582506689998,3458520.1421022415,3405312.139916053,0.0,2026-08-01T14:04:14.150769+08:00,2026-08-01T14:11:00.235237+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm +confirm,rdma_confirm_i65536_o1024_c16,2,COMPLETED,65536,1024,16,16,406.09330375201534,2582.1061079114043,40.34540793611569,2622.45151584752,336924.2522477143,356.06784952074946,99.48923731257209,71.66656730160658,86.33499788736604,89.31414888004788,143.33313460321313,0.19167460586668467,3457155.7233657837,3403968.712237079,0.0,2026-08-01T14:11:34.981654+08:00,2026-08-01T14:18:21.074958+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm +confirm,rdma_confirm_i65536_o1024_c64,1,COMPLETED,65536,1024,64,64,1619.8975476099877,2589.2402925038787,40.456879570373104,2629.6971720742517,1506265.7001026294,418.57784627036636,99.6722631148773,72.19382480758325,86.71108451788668,89.82468221963666,144.38764961516648,0.070112013789426,3483446.6699664593,3429855.182736206,0.0,2026-08-01T14:19:02.582676+08:00,2026-08-01T14:46:02.480224+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm +confirm,rdma_confirm_i65536_o1024_c64,2,COMPLETED,65536,1024,64,64,1620.2422464910196,2588.689443867829,40.44827256043483,2629.137716428264,1506724.735757988,418.5783493891426,99.67225809785734,72.18433270089784,86.67873733217695,89.23672463088391,144.36866540179568,0.005496635275729702,3483400.1801579,3429809.4081554706,0.0,2026-08-01T14:46:37.465815+08:00,2026-08-01T15:13:37.708061+08:00,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/confirm diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_model.json b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_model.json new file mode 100644 index 0000000..47bd20a --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_model.json @@ -0,0 +1,205 @@ +{ + "generated_at": "2026-08-01T15:14:11.558999+08:00", + "formula": "rail_gbps ~= input_tps * bytes_per_input_token_per_rail * 8 / 1e9", + "plateau_gain_pct": 5.0, + "target_rail_gbps": 400.0, + "practical_rail_gbps": 360.0, + "scout": [ + { + "stage": "scout", + "concurrency": 1, + "repetitions": 1, + "input_tps": 2709.6368150803128, + "output_tps": 0.04134577659729481, + "total_tps": 2709.67816085691, + "ttft_p95_ms": 24167.605170048773, + "tpot_p95_ms": 0.0, + "gpu_util_mean_pct": 93.74897875816994, + "rail_xmit_avg_gbps": 70.9712746848062, + "rail_xmit_p95_gbps": 85.5972372945909, + "rail_xmit_max_gbps": 86.80566056858429, + "dual_rail_xmit_avg_gbps": 141.9425493696124, + "rail_imbalance_pct": 0.5207119247408836, + "xmit_bytes_per_input_token_per_rail": 3137732.762954712, + "xmit_bytes_per_total_token_per_rail": 3137684.8856828967, + "rdma_error_delta": 0.0 + }, + { + "stage": "scout", + "concurrency": 4, + "repetitions": 1, + "input_tps": 2930.491100122512, + "output_tps": 0.044715745546302976, + "total_tps": 2930.535815868058, + "ttft_p95_ms": 86526.69315790116, + "tpot_p95_ms": 0.0, + "gpu_util_mean_pct": 97.4494891826923, + "rail_xmit_avg_gbps": 78.27464182068744, + "rail_xmit_p95_gbps": 86.81919268342193, + "rail_xmit_max_gbps": 88.85894624954848, + "dual_rail_xmit_avg_gbps": 156.5492836413749, + "rail_imbalance_pct": 0.2703571902588796, + "xmit_bytes_per_input_token_per_rail": 3298012.1388320923, + "xmit_bytes_per_total_token_per_rail": 3297961.815928407, + "rdma_error_delta": 0.0 + }, + { + "stage": "scout", + "concurrency": 16, + "repetitions": 1, + "input_tps": 2983.771164628114, + "output_tps": 0.045528734811830354, + "total_tps": 2983.816693362926, + "ttft_p95_ms": 336717.4506017618, + "tpot_p95_ms": 0.0, + "gpu_util_mean_pct": 99.33137833282255, + "rail_xmit_avg_gbps": 79.89685137869246, + "rail_xmit_p95_gbps": 86.416508483831, + "rail_xmit_max_gbps": 88.20167661610235, + "dual_rail_xmit_avg_gbps": 159.79370275738495, + "rail_imbalance_pct": 0.1149304623247205, + "xmit_bytes_per_input_token_per_rail": 3335858.008945465, + "xmit_bytes_per_total_token_per_rail": 3335807.108568442, + "rdma_error_delta": 0.0 + }, + { + "stage": "scout", + "concurrency": 32, + "repetitions": 1, + "input_tps": 2983.9209393567885, + "output_tps": 0.04553102019282209, + "total_tps": 2983.9664703769813, + "ttft_p95_ms": 671139.2772838357, + "tpot_p95_ms": 0.0, + "gpu_util_mean_pct": 99.49393723047987, + "rail_xmit_avg_gbps": 79.92852806348517, + "rail_xmit_p95_gbps": 86.22587499587476, + "rail_xmit_max_gbps": 88.21644913628985, + "dual_rail_xmit_avg_gbps": 159.85705612697035, + "rail_imbalance_pct": 0.10091204027725086, + "xmit_bytes_per_input_token_per_rail": 3343929.3806581497, + "xmit_bytes_per_total_token_per_rail": 3343878.357123648, + "rdma_error_delta": 0.0 + }, + { + "stage": "scout", + "concurrency": 64, + "repetitions": 1, + "input_tps": 2991.2818470623424, + "output_tps": 0.04564333873080967, + "total_tps": 2991.327490401073, + "ttft_p95_ms": 1337131.1103800863, + "tpot_p95_ms": 0.0, + "gpu_util_mean_pct": 99.63928846991146, + "rail_xmit_avg_gbps": 79.9743866369171, + "rail_xmit_p95_gbps": 85.93278751515446, + "rail_xmit_max_gbps": 88.32212536165416, + "dual_rail_xmit_avg_gbps": 159.9487732738342, + "rail_imbalance_pct": 0.03496630506355109, + "xmit_bytes_per_input_token_per_rail": 3339597.901974201, + "xmit_bytes_per_total_token_per_rail": 3339546.9445318105, + "rdma_error_delta": 0.0 + } + ], + "confirm": [ + { + "stage": "confirm", + "concurrency": 4, + "repetitions": 2, + "input_tps": 2072.2412386841634, + "output_tps": 32.37876935444005, + "total_tps": 2104.6200080386034, + "ttft_p95_ms": 86364.19230348256, + "tpot_p95_ms": 95.63100270686752, + "gpu_util_mean_pct": 98.2331583326596, + "rail_xmit_avg_gbps": 57.332462905232674, + "rail_xmit_p95_gbps": 86.27849297798633, + "rail_xmit_max_gbps": 88.76621317376211, + "dual_rail_xmit_avg_gbps": 114.66492581046535, + "rail_imbalance_pct": 2.0049749155861494, + "xmit_bytes_per_input_token_per_rail": 3438554.733493805, + "xmit_bytes_per_total_token_per_rail": 3385653.891440054, + "rdma_error_delta": 0.0 + }, + { + "stage": "confirm", + "concurrency": 16, + "repetitions": 2, + "input_tps": 2582.1341992166654, + "output_tps": 40.345846862760396, + "total_tps": 2622.480046079426, + "ttft_p95_ms": 336851.4825689708, + "tpot_p95_ms": 356.24346312779323, + "gpu_util_mean_pct": 99.4390366116125, + "rail_xmit_avg_gbps": 71.64370371931446, + "rail_xmit_p95_gbps": 86.29922510297757, + "rail_xmit_max_gbps": 89.12703945922908, + "dual_rail_xmit_avg_gbps": 143.28740743862892, + "rail_imbalance_pct": 0.15289021546679232, + "xmit_bytes_per_input_token_per_rail": 3457837.9327340126, + "xmit_bytes_per_total_token_per_rail": 3404640.426076566, + "rdma_error_delta": 0.0 + }, + { + "stage": "confirm", + "concurrency": 64, + "repetitions": 2, + "input_tps": 2588.964868185854, + "output_tps": 40.45257606540397, + "total_tps": 2629.4174442512576, + "ttft_p95_ms": 1506495.2179303085, + "tpot_p95_ms": 418.5780978297545, + "gpu_util_mean_pct": 99.67226060636732, + "rail_xmit_avg_gbps": 72.18907875424054, + "rail_xmit_p95_gbps": 86.69491092503182, + "rail_xmit_max_gbps": 89.53070342526028, + "dual_rail_xmit_avg_gbps": 144.37815750848108, + "rail_imbalance_pct": 0.03780432453257785, + "xmit_bytes_per_input_token_per_rail": 3483423.4250621796, + "xmit_bytes_per_total_token_per_rail": 3429832.295445838, + "rdma_error_delta": 0.0 + } + ], + "selected_confirm_concurrencies": [ + 4, + 16, + 64 + ], + "observed_plateau_concurrency": 16, + "best_scout_bandwidth_concurrency": 64, + "linear_gbps_per_input_tps": 0.026656368057740153, + "modeled_bytes_per_input_token_per_rail": 3332046.007217519, + "required_input_tps_for_400g_per_rail": 15005.795205617025, + "required_input_tps_for_360g_per_rail": 13505.215685055322, + "required_input_tps_for_400g_dual_rail_aggregate": 7502.897602808513, + "bandwidth_saturation_fit": { + "asymptote": 80.32102454669293, + "half_saturation_c": 0.12589254117941673, + "rmse": 0.27652726609534695 + }, + "input_tps_saturation_fit": { + "asymptote": 3003.9418335884134, + "half_saturation_c": 0.11220184543019636, + "rmse": 7.680357081457988 + }, + "best_confirm_point": { + "stage": "confirm", + "concurrency": 64, + "repetitions": 2, + "input_tps": 2588.964868185854, + "output_tps": 40.45257606540397, + "total_tps": 2629.4174442512576, + "ttft_p95_ms": 1506495.2179303085, + "tpot_p95_ms": 418.5780978297545, + "gpu_util_mean_pct": 99.67226060636732, + "rail_xmit_avg_gbps": 72.18907875424054, + "rail_xmit_p95_gbps": 86.69491092503182, + "rail_xmit_max_gbps": 89.53070342526028, + "dual_rail_xmit_avg_gbps": 144.37815750848108, + "rail_imbalance_pct": 0.03780432453257785, + "xmit_bytes_per_input_token_per_rail": 3483423.4250621796, + "xmit_bytes_per_total_token_per_rail": 3429832.295445838, + "rdma_error_delta": 0.0 + }, + "verdict": "COMPUTE_OR_MODEL_THROUGHPUT_LIMITED_BEFORE_RDMA_SATURATION" +} diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_report.md b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_report.md new file mode 100644 index 0000000..057fff2 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/rdma_demand_report.md @@ -0,0 +1,37 @@ +# Phase 2.5 RDMA Demand Model + +- Generated: `2026-08-01T15:14:11.558999+08:00` +- Verdict: `COMPUTE_OR_MODEL_THROUGHPUT_LIMITED_BEFORE_RDMA_SATURATION` +- Model: `rail_gbps ~= input_tps x 3.332 MB/token/rail x 8` +- Required Input TPS for 400G per rail: `15006` +- Required Input TPS for 360G per rail: `13505` +- Required Input TPS for 400G dual-rail aggregate: `7503` +- Fitted rail bandwidth ceiling: `80.32 Gbit/s` +- Observed plateau concurrency: `16` + +## Scout: 64K -> 1 + +| C | Input TPS | Rail avg | Rail P95 | Dual-rail avg | MB/input-token/rail | GPU util | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 2709.64 | 70.97 | 85.60 | 141.94 | 3.138 | 93.75 | +| 4 | 2930.49 | 78.27 | 86.82 | 156.55 | 3.298 | 97.45 | +| 16 | 2983.77 | 79.90 | 86.42 | 159.79 | 3.336 | 99.33 | +| 32 | 2983.92 | 79.93 | 86.23 | 159.86 | 3.344 | 99.49 | +| 64 | 2991.28 | 79.97 | 85.93 | 159.95 | 3.340 | 99.64 | + +## Confirm: 64K -> 1K + +| C | Reps | Input TPS | Output TPS | Rail avg | Rail P95 | Dual-rail avg | TTFT P95 ms | TPOT P95 ms | +|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 | 2 | 2072.24 | 32.38 | 57.33 | 86.28 | 114.66 | 86364.19 | 95.63 | +| 16 | 2 | 2582.13 | 40.35 | 71.64 | 86.30 | 143.29 | 336851.48 | 356.24 | +| 64 | 2 | 2588.96 | 40.45 | 72.19 | 86.69 | 144.38 | 1506495.22 | 418.58 | + +## Evidence + +- Per-case metrics: `rdma_case_metrics.csv` +- Machine-readable model: `rdma_demand_model.json` +- Scout HCA source: `scout/{head,worker}/rdma.csv` +- Confirm HCA source: `confirm/{head,worker}/rdma.csv` +- Exact windows: `scout/case_windows.csv`, `confirm/case_windows.csv` +- Benchmark summaries: `scout/bench_summary.csv`, `confirm/bench_summary.csv` diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/recommendation.env b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/recommendation.env new file mode 100644 index 0000000..fa0ebb9 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/recommendation.env @@ -0,0 +1 @@ +CONFIRM_CONCURRENCIES='4 16 64' diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/run_manifest.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/run_manifest.txt new file mode 100644 index 0000000..da3b529 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/run_manifest.txt @@ -0,0 +1,13 @@ +run_id=dsv4pro-phase2_5-20260801-130007 +experiment=dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling +created_at=2026-08-01T13:00:07+0800 +git_commit=c5fa700c50c0cc1b9e20d4d2d3f03bd485ba0dbf +git_dirty=0 +isl=65536 +scout_osl=1 +confirm_osl=1024 +scout_concurrencies=1 4 16 32 64 +scout_repetitions=1 +confirm_repetitions=2 +target_rail_gbps=400 +practical_rail_gbps=360 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/confirm.tsv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/confirm.tsv new file mode 100644 index 0000000..d6b9324 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/confirm.tsv @@ -0,0 +1,4 @@ +# case_id stage isl osl concurrency prompt_multiplier min_prompts warmup_requests note +rdma_confirm_i65536_o1024_c4 rdma_confirm 65536 1024 4 1 1 0 Phase 2.5 confirm: correlate model token rate with dual-rail RDMA +rdma_confirm_i65536_o1024_c16 rdma_confirm 65536 1024 16 1 1 0 Phase 2.5 confirm: correlate model token rate with dual-rail RDMA +rdma_confirm_i65536_o1024_c64 rdma_confirm 65536 1024 64 1 1 0 Phase 2.5 confirm: correlate model token rate with dual-rail RDMA diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/scout.tsv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/scout.tsv new file mode 100644 index 0000000..044b313 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scenarios/scout.tsv @@ -0,0 +1,6 @@ +# case_id stage isl osl concurrency prompt_multiplier min_prompts warmup_requests note +rdma_scout_i65536_o1_c1 rdma_scout 65536 1 1 1 1 0 Phase 2.5 scout: correlate model token rate with dual-rail RDMA +rdma_scout_i65536_o1_c4 rdma_scout 65536 1 4 1 1 0 Phase 2.5 scout: correlate model token rate with dual-rail RDMA +rdma_scout_i65536_o1_c16 rdma_scout 65536 1 16 1 1 0 Phase 2.5 scout: correlate model token rate with dual-rail RDMA +rdma_scout_i65536_o1_c32 rdma_scout 65536 1 32 1 1 0 Phase 2.5 scout: correlate model token rate with dual-rail RDMA +rdma_scout_i65536_o1_c64 rdma_scout 65536 1 64 1 1 0 Phase 2.5 scout: correlate model token rate with dual-rail RDMA diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/bench_summary.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/bench_summary.csv new file mode 100644 index 0000000..030e816 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/bench_summary.csv @@ -0,0 +1,6 @@ +run_id,suite,case_id,role,stage,repetition,isl,osl,concurrency,num_prompts,warmup_requests,status,error_type,exit_code,started_at,ended_at,elapsed_s,measurement_started_at,measurement_ended_at,measurement_duration_s,measurement_window_source,completed,failed,duration_s,actual_concurrency,peak_concurrent_requests,total_input_tokens,total_output_tokens,request_throughput,input_token_throughput,output_token_throughput,total_token_throughput,peak_output_token_throughput,e2e_mean_ms,e2e_p50_ms,e2e_p95_ms,e2e_p99_ms,ttft_mean_ms,ttft_p50_ms,ttft_p95_ms,ttft_p99_ms,tpot_mean_ms,tpot_p50_ms,tpot_p95_ms,tpot_p99_ms,itl_mean_ms,itl_p50_ms,itl_p95_ms,itl_p99_ms,bench_file,bench_log,phase2_bench_run +fixed_rdma_scout_i65536_o1_c1,fixed,rdma_scout_i65536_o1_c1,,rdma_scout,1,65536,1,1,1,0,COMPLETED,,0,2026-08-01T13:05:54+0800,2026-08-01T13:06:53+0800,59.0,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,24.1862671909621,bench_main_marker_plus_duration,1,0,24.1862671909621,0.9992315809294645,,65536,1,0.04134577659729481,2709.6368150803128,0.04134577659729481,2709.67816085691,,24167.682002007496,24167.682002007496,24167.682002007496,24167.682002007496,24167.605170048773,24167.605170048773,24167.605170048773,24167.605170048773,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c1/cases/rdma_scout_i65536_o1_c1/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c1/cases/rdma_scout_i65536_o1_c1/rep1/bench.log,fixed_rdma_scout_i65536_o1_c1 +fixed_rdma_scout_i65536_o1_c16,fixed,rdma_scout_i65536_o1_c16,,rdma_scout,1,65536,1,16,16,0,COMPLETED,,0,2026-08-01T13:09:09+0800,2026-08-01T13:15:34+0800,385.0,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,351.42641380499117,bench_main_marker_plus_duration,16,0,351.42641380499117,8.628873239905923,,1048576,16,0.045528734811830354,2983.771164628114,0.045528734811830354,2983.816693362926,,189525.8736173746,189769.315014506,336717.50638925005,348428.8343418157,189525.84591050618,189769.29299000767,336717.4506017618,348428.77078671294,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c16/cases/rdma_scout_i65536_o1_c16/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c16/cases/rdma_scout_i65536_o1_c16/rep1/bench.log,fixed_rdma_scout_i65536_o1_c16 +fixed_rdma_scout_i65536_o1_c32,fixed,rdma_scout_i65536_o1_c32,,rdma_scout,1,65536,1,32,32,0,COMPLETED,,0,2026-08-01T13:15:40+0800,2026-08-01T13:27:58+0800,738.0,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,702.8175486620166,bench_main_marker_plus_duration,32,0,702.8175486620166,16.63411701870453,,2097152,32,0.04553102019282209,2983.9209393567885,0.04553102019282209,2983.9664703769813,,365335.9171013453,365369.92930600536,671139.3307480408,696643.7062339701,365335.8620414692,365369.8761375272,671139.2772838357,696643.6536653765,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c32/cases/rdma_scout_i65536_o1_c32/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c32/cases/rdma_scout_i65536_o1_c32/rep1/bench.log,fixed_rdma_scout_i65536_o1_c32 +fixed_rdma_scout_i65536_o1_c4,fixed,rdma_scout_i65536_o1_c4,,rdma_scout,1,65536,1,4,4,0,COMPLETED,,0,2026-08-01T13:06:59+0800,2026-08-01T13:09:02+0800,123.0,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,89.45394851703895,bench_main_marker_plus_duration,4,0,89.45394851703895,2.5881014457616573,,262144,4,0.044715745546302976,2930.491100122512,0.044715745546302976,2930.535815868058,,57878.973371509346,58521.736774011515,86526.744662947,88841.4666661853,57878.91561826109,58521.6828455159,86526.69315790116,88841.41537077782,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c4/cases/rdma_scout_i65536_o1_c4/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c4/cases/rdma_scout_i65536_o1_c4/rep1/bench.log,fixed_rdma_scout_i65536_o1_c4 +fixed_rdma_scout_i65536_o1_c64,fixed,rdma_scout_i65536_o1_c64,,rdma_scout,1,65536,1,64,64,0,COMPLETED,,0,2026-08-01T13:28:04+0800,2026-08-01T13:52:01+0800,1437.0,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,1402.17612864502,bench_main_marker_plus_duration,64,0,1402.17612864502,32.732392604632125,,4194304,64,0.04564333873080967,2991.2818470623424,0.04564333873080967,2991.327490401073,,717134.0553695618,717609.0070659702,1337131.160498035,1390460.9022671543,717134.0051560618,717608.9713889814,1337131.1103800863,1390460.8547802733,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c64/cases/rdma_scout_i65536_o1_c64/rep1/bench.jsonl,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c64/cases/rdma_scout_i65536_o1_c64/rep1/bench.log,fixed_rdma_scout_i65536_o1_c64 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_rdma_summary.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_rdma_summary.csv new file mode 100644 index 0000000..d8f6407 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_rdma_summary.csv @@ -0,0 +1,21 @@ +phase2_bench_run,case_id,role,stage,repetition,status,started_at,ended_at,duration_s,window_source,node,hca,samples,xmit_bytes,rcv_bytes,xmit_gbps,rcv_gbps,port_xmit_wait_delta,port_xmit_discards_delta,port_rcv_errors_delta,req_transport_retries_exceeded_delta,req_rnr_retries_exceeded_delta,roce_adp_retrans_delta,roce_adp_retrans_to_delta,np_ecn_marked_roce_packets_delta,np_cnp_sent_delta,rp_cnp_handled_delta +fixed_rdma_scout_i65536_o1_c1,rdma_scout_i65536_o1_c1,,rdma_scout,1,COMPLETED,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,23.006126848,bench_main_marker_plus_duration,head,mlx5_0,22,203533427700.0,203651710996.0,70.77538224308064,70.81651330239592,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c1,rdma_scout_i65536_o1_c1,,rdma_scout,1,COMPLETED,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,23.006126848,bench_main_marker_plus_duration,head,mlx5_3,22,204100301340.0,204100542584.0,70.97250317308169,70.97258706169157,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c1,rdma_scout_i65536_o1_c1,,rdma_scout,1,COMPLETED,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,23.352313088,bench_main_marker_plus_duration,worker,mlx5_0,23,207229229632.0,207304460944.0,70.99227518955745,71.01804781832155,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c1,rdma_scout_i65536_o1_c1,,rdma_scout,1,COMPLETED,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,23.352313088,bench_main_marker_plus_duration,worker,mlx5_3,23,207674858740.0,207674728976.0,71.14493813350504,71.14489367915073,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c16,rdma_scout_i65536_o1_c16,,rdma_scout,1,COMPLETED,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,350.585191424,bench_main_marker_plus_duration,head,mlx5_0,321,3499319374812.0,3499321894556.0,79.850934047694,79.85099154570729,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c16,rdma_scout_i65536_o1_c16,,rdma_scout,1,COMPLETED,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,350.585191424,bench_main_marker_plus_duration,head,mlx5_3,321,3499670238252.0,3499687020088.0,79.85894039704549,79.85932334159443,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c16,rdma_scout_i65536_o1_c16,,rdma_scout,1,COMPLETED,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,349.898471936,bench_main_marker_plus_duration,worker,mlx5_0,331,3496131787252.0,3496130977288.0,79.93477120166395,79.93475268282916,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c16,rdma_scout_i65536_o1_c16,,rdma_scout,1,COMPLETED,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,349.898471936,bench_main_marker_plus_duration,worker,mlx5_3,331,3496481190036.0,3496480578468.0,79.94275986836642,79.94274588561318,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c32,rdma_scout_i65536_o1_c32,,rdma_scout,1,COMPLETED,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,701.184411136,bench_main_marker_plus_duration,head,mlx5_0,641,7008726918884.0,7008731821472.0,79.96443511947507,79.96449105440941,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c32,rdma_scout_i65536_o1_c32,,rdma_scout,1,COMPLETED,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,701.184411136,bench_main_marker_plus_duration,head,mlx5_3,641,7009132360248.0,7009232426556.0,79.9690609081556,79.97020259135803,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c32,rdma_scout_i65536_o1_c32,,rdma_scout,1,COMPLETED,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,702.616064768,bench_main_marker_plus_duration,worker,mlx5_0,663,7016359452164.0,7016357965840.0,79.88840339972316,79.88838647640956,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c32,rdma_scout_i65536_o1_c32,,rdma_scout,1,COMPLETED,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,702.616064768,bench_main_marker_plus_duration,worker,mlx5_3,663,7016694022728.0,7016692492508.0,79.89221282658687,79.89219540347258,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c4,rdma_scout_i65536_o1_c4,,rdma_scout,1,COMPLETED,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,88.716937728,bench_main_marker_plus_duration,head,mlx5_0,82,866863327600.0,866884788736.0,78.16891338226691,78.17084862814438,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c4,rdma_scout_i65536_o1_c4,,rdma_scout,1,COMPLETED,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,88.716937728,bench_main_marker_plus_duration,head,mlx5_3,82,867282690684.0,867282482924.0,78.20672921268122,78.20671047804,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c4,rdma_scout_i65536_o1_c4,,rdma_scout,1,COMPLETED,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,88.006032896,bench_main_marker_plus_duration,worker,mlx5_0,84,861825370952.0,861825497804.0,78.34239018322312,78.34240171443258,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c4,rdma_scout_i65536_o1_c4,,rdma_scout,1,COMPLETED,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,88.006032896,bench_main_marker_plus_duration,worker,mlx5_3,84,862244987252.0,862244674092.0,78.38053450457852,78.38050603743919,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c64,rdma_scout_i65536_o1_c64,,rdma_scout,1,COMPLETED,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,1401.218045952,bench_main_marker_plus_duration,head,mlx5_0,1280,14005295392512.0,14005304292920.0,79.96069096010922,79.96074177537255,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c64,rdma_scout_i65536_o1_c64,,rdma_scout,1,COMPLETED,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,1401.218045952,bench_main_marker_plus_duration,head,mlx5_3,1280,14005533397520.0,14005553945192.0,79.96204980648541,79.96216711968765,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c64,rdma_scout_i65536_o1_c64,,rdma_scout,1,COMPLETED,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,1401.136956672,bench_main_marker_plus_duration,worker,mlx5_0,1321,14008943976736.0,14008941122316.0,79.9861507329604,79.98613443522456,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fixed_rdma_scout_i65536_o1_c64,rdma_scout_i65536_o1_c64,,rdma_scout,1,COMPLETED,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,1401.136956672,bench_main_marker_plus_duration,worker,mlx5_3,1321,14009382587800.0,14009379963032.0,79.98865504811339,79.98864006160981,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_windows.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_windows.csv new file mode 100644 index 0000000..ff910e3 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/case_windows.csv @@ -0,0 +1,6 @@ +phase2_bench_run,case_id,role,stage,repetition,status,started_at,ended_at,started_ns,ended_ns,duration_s,window_source,meta_path +fixed_rdma_scout_i65536_o1_c1,rdma_scout_i65536_o1_c1,,rdma_scout,1,COMPLETED,2026-08-01T13:06:26.217761+08:00,2026-08-01T13:06:50.404028+08:00,1785560786217761024,1785560810404027904,24.18626688,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c1/cases/rdma_scout_i65536_o1_c1/rep1/meta.json +fixed_rdma_scout_i65536_o1_c16,rdma_scout_i65536_o1_c16,,rdma_scout,1,COMPLETED,2026-08-01T13:09:39.903596+08:00,2026-08-01T13:15:31.330010+08:00,1785560979903596032,1785561331330009856,351.426413824,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c16/cases/rdma_scout_i65536_o1_c16/rep1/meta.json +fixed_rdma_scout_i65536_o1_c32,rdma_scout_i65536_o1_c32,,rdma_scout,1,COMPLETED,2026-08-01T13:16:12.166763+08:00,2026-08-01T13:27:54.984312+08:00,1785561372166763008,1785562074984312064,702.817549056,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c32/cases/rdma_scout_i65536_o1_c32/rep1/meta.json +fixed_rdma_scout_i65536_o1_c4,rdma_scout_i65536_o1_c4,,rdma_scout,1,COMPLETED,2026-08-01T13:07:30.521113+08:00,2026-08-01T13:08:59.975062+08:00,1785560850521112832,1785560939975062016,89.453949184,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c4/cases/rdma_scout_i65536_o1_c4/rep1/meta.json +fixed_rdma_scout_i65536_o1_c64,rdma_scout_i65536_o1_c64,,rdma_scout,1,COMPLETED,2026-08-01T13:28:36.631968+08:00,2026-08-01T13:51:58.808097+08:00,1785562116631968000,1785563518808096768,1402.176128768,bench_main_marker_plus_duration,/data/hzy/sskj/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_rdma_demand_modeling/results/dsv4pro-phase2_5-20260801-130007/scout/bench/fixed_rdma_scout_i65536_o1_c64/cases/rdma_scout_i65536_o1_c64/rep1/meta.json diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/collector_status.csv b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/collector_status.csv new file mode 100644 index 0000000..829d2d9 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/collector_status.csv @@ -0,0 +1,37 @@ +wall_time_ns,node,collector,status,pid +1785560744455556276,head,gpu_samples.csv,STARTED,2168337 +1785560744470800548,head,rdma.csv,STARTED,2168348 +1785560744485157848,head,dcgm_dmon.log,STARTED,2168362 +1785560744499919157,head,mpstat.log,STARTED,2168380 +1785560744514378735,head,pidstat.log,STARTED,2168400 +1785560744529055413,head,sar_net.log,STARTED,2168428 +1785560744543220536,head,perf_stat.log,STARTED,2168451 +1785560744559383367,head,docker_top.log,STARTED,2168486 +1785560744579614511,head,numa_samples.csv,STARTED,2168561 +1785560744602206477,worker,gpu_samples.csv,STARTED,2168604 +1785560744619402981,worker,rdma.csv,STARTED,2168664 +1785560744634896197,worker,dcgm_dmon.log,STARTED,2168717 +1785560744650285593,worker,mpstat.log,STARTED,2168763 +1785560744666005300,worker,pidstat.log,STARTED,2168775 +1785560744681017748,worker,sar_net.log,STARTED,2168787 +1785560744694432153,worker,perf_stat.log,STARTED,2168796 +1785560744709522681,worker,docker_top.log,STARTED,2168808 +1785560744723983222,worker,numa_samples.csv,STARTED,2168818 +1785563532345901412,head,gpu_samples.csv,STOPPED,2168337 +1785563532350635898,head,rdma.csv,STOPPED,2168348 +1785563532357352405,head,dcgm_dmon.log,STOPPED,2168362 +1785563532362419425,head,mpstat.log,STOPPED,2168380 +1785563532367417490,head,pidstat.log,STOPPED,2168400 +1785563532372197961,head,sar_net.log,STOPPED,2168428 +1785563532541034088,head,perf_stat.log,STOPPED,2168451 +1785563532545727806,head,docker_top.log,STOPPED,2168486 +1785563532550309298,head,numa_samples.csv,STOPPED,2168561 +1785563532554473337,worker,gpu_samples.csv,STOPPED,2168604 +1785563532558689660,worker,rdma.csv,STOPPED,2168664 +1785563532562663636,worker,dcgm_dmon.log,STOPPED,2168717 +1785563532566892350,worker,mpstat.log,STOPPED,2168763 +1785563532571076626,worker,pidstat.log,STOPPED,2168775 +1785563532575164431,worker,sar_net.log,STOPPED,2168787 +1785563532579199145,worker,perf_stat.log,STOPPED,2168796 +1785563532583371937,worker,docker_top.log,STOPPED,2168808 +1785563532587359055,worker,numa_samples.csv,STOPPED,2168818 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_nccl_transport.log b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_nccl_transport.log new file mode 100644 index 0000000..7a2df26 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_nccl_transport.log @@ -0,0 +1,1012 @@ +/usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). +/sgl-workspace/sglang/python/sglang/launch_server.py:56: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-01 05:00:43] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:43] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:43] Breakable CUDA graph is incompatible with DeepSeek-V4 (heavy capture-pool memory pressure); disabling prefill CUDA graph. +[2026-08-01 05:00:43] Use dsv4 attention backend for DeepseekV4ForCausalLM, setting page_size to 256. +[2026-08-01 05:00:43] Setting swa_full_tokens_ratio to 0.1 for DeepseekV4ForCausalLM. +[2026-08-01 05:00:43] Setting KV cache dtype to fp8_e4m3 for DeepseekV4ForCausalLM. +[2026-08-01 05:00:43] Use flashinfer_mxfp4 as MoE runner backend on SM120 for DeepseekV4 +[2026-08-01 05:00:46] server_args=ServerArgs(model_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=None, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.9, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=8192, enable_dynamic_chunking=False, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, retraction_policy='length', schedule_conservativeness=1.0, page_size=256, swa_full_tokens_ratio=0.1, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, enable_page_major_kv_layout=False, enable_unified_memory=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, nccl_port=None, dist_timeout=None, dist_init_addr='10.101.0.11:20002', nnodes=2, node_rank=0, tp_size=16, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=1, load_balance_method='round_robin', attn_cp_size=1, moe_dp_size=1, enable_prefill_cp=False, cp_strategy=None, enable_dsa_cache_layer_split=False, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_dp_attention=False, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=False, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=681188556, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, host='0.0.0.0', port=30002, fastapi_root_path='', smg_grpc_mode=False, grpc_mode=False, grpc_port=None, skip_server_warmup=False, warmups=None, enable_http2=False, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, api_key=None, admin_api_key=None, served_model_name='/data/hf_models/DeepSeek-V4-Pro', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, default_chat_template_kwargs=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser=None, tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, smg_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, attention_backend='dsv4', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='auto', bf16_gemm_backend='auto', dsa_prefill_backend=None, dsa_decode_backend=None, dsa_paged_mqa_logits_backend='auto', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, mamba_backend='triton', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=64, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], tc_compiler='eager', full_prefill_max_req=None), prefill=PhaseConfig(backend='disabled', max_bs=8192, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], tc_compiler='eager', full_prefill_max_req=None)), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=64, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, torchao_config='', speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_dspark_block_size=None, speculative_dspark_sps_table_path=None, speculative_dspark_confidence_sts_path=None, speculative_dspark_align_verify_tokens_to_graph_tier=False, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_window_size=None, speculative_moe_runner_backend='flashinfer_mxfp4', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=2, moe_a2a_backend='none', moe_runner_backend='flashinfer_mxfp4', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', fuseep_mode=2, deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_waterfill=False, ep_join_mode=None, ep_join_rank_offset=0, elastic_ep_initial_size=None, max_ep_size=None, elastic_ep_scale_timeout=600, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=False, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, enable_mamba_cache_stochastic_rounding=False, mamba_cache_philox_rounds=0, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', uses_mamba_radix_cache=False, mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, enable_mm_global_cache=False, disable_fast_image_processor=False, mm_feature_transport='cpu', keep_mm_feature_on_device=False, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', enable_lmcache=False, lmcache_config_file=None, enable_flexkv=False, flexkv_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, dllm_fdfo=True, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_attempts=0, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, forward_hooks=None, msprobe_dump_config=None) +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:46] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:46] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:48] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:49] No HuggingFace chat template found +[2026-08-01 05:00:49] No chat template found, defaulting to 'string' content format +[2026-08-01 05:01:00 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:00 TP0 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP4 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP4 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP2 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP2 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP7 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP7 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP5 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP5 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP1 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP1 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP6 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP6 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:01 TP3 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:01 TP3 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:02] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:02 TP0 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:03 TP0 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:03 TP0 EP0] Init torch distributed begin. +[2026-08-01 05:01:03 TP4 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP2 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP5 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP7 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP4 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:03 TP4 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:03 TP2 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:03 TP2 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:03 TP3 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP6 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP1 EP0] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:01:03 TP2 EP0] Init torch distributed begin. +[2026-08-01 05:01:03 TP4 EP0] Init torch distributed begin. +[2026-08-01 05:01:03 TP5 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:03 TP5 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:03 TP7 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:03 TP7 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:03 TP3 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:03 TP3 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:04 TP7 EP0] Init torch distributed begin. +[2026-08-01 05:01:04 TP5 EP0] Init torch distributed begin. +[2026-08-01 05:01:04 TP1 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:04 TP1 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:04 TP6 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:01:04 TP6 EP0] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:01:04 TP3 EP0] Init torch distributed begin. +[2026-08-01 05:01:04 TP1 EP0] Init torch distributed begin. +[2026-08-01 05:01:04 TP6 EP0] Init torch distributed begin. +[2026-08-01 05:01:10 TP0 EP0] sglang is using nccl==2.28.9 +gpu1:236:236 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:236:236 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:236:236 [0] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:236:236 [0] NCCL INFO cudaDriverVersion 13000 +gpu1:236:236 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:241:241 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:241:241 [5] NCCL INFO cudaDriverVersion 13000 +gpu1:237:237 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:237:237 [1] NCCL INFO cudaDriverVersion 13000 +gpu1:241:241 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:241:241 [5] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:241:241 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:237:237 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:238:238 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:238:238 [2] NCCL INFO cudaDriverVersion 13000 +gpu1:237:237 [1] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:237:237 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:242:242 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:242:242 [6] NCCL INFO cudaDriverVersion 13000 +gpu1:240:240 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:240:240 [4] NCCL INFO cudaDriverVersion 13000 +gpu1:238:238 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:238:238 [2] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:238:238 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:242:242 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:240:240 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:242:242 [6] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:242:242 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:240:240 [4] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:240:240 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:239:239 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:239:239 [3] NCCL INFO cudaDriverVersion 13000 +gpu1:243:243 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu1:243:243 [7] NCCL INFO cudaDriverVersion 13000 +gpu1:239:239 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:243:243 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:239:239 [3] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:239:239 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:243:243 [7] NCCL INFO Bootstrap: Using eth0:10.101.0.11<0> +gpu1:243:243 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu1:240:240 [4] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:240:240 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:240:240 [4] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:238:238 [2] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:238:238 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:238:238 [2] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:242:242 [6] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:242:242 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:242:242 [6] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:243:243 [7] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:243:243 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:243:243 [7] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:241:241 [5] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:241:241 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:241:241 [5] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:239:239 [3] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:239:239 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:239:239 [3] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:243:243 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:243:243 [7] NCCL INFO Initialized NET plugin IB +gpu1:243:243 [7] NCCL INFO Assigned NET plugin IB to comm +gpu1:243:243 [7] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:243:243 [7] NCCL INFO Using network IB +gpu1:237:237 [1] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:237:237 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:237:237 [1] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:241:241 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:241:241 [5] NCCL INFO Initialized NET plugin IB +gpu1:241:241 [5] NCCL INFO Assigned NET plugin IB to comm +gpu1:241:241 [5] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:241:241 [5] NCCL INFO Using network IB +gpu1:243:243 [7] NCCL INFO DMA-BUF is available on GPU device 7 +gpu1:243:243 [7] NCCL INFO ncclCommInitRank comm 0x1a399720 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x61e0ecdb3f010655 - Init START +gpu1:241:241 [5] NCCL INFO DMA-BUF is available on GPU device 5 +gpu1:241:241 [5] NCCL INFO ncclCommInitRank comm 0x4e055680 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x61e0ecdb3f010655 - Init START +gpu1:237:237 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:237:237 [1] NCCL INFO Initialized NET plugin IB +gpu1:237:237 [1] NCCL INFO Assigned NET plugin IB to comm +gpu1:237:237 [1] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:237:237 [1] NCCL INFO Using network IB +gpu1:237:237 [1] NCCL INFO DMA-BUF is available on GPU device 1 +gpu1:237:237 [1] NCCL INFO ncclCommInitRank comm 0x3a507100 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x61e0ecdb3f010655 - Init START +gpu1:238:238 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:238:238 [2] NCCL INFO Initialized NET plugin IB +gpu1:238:238 [2] NCCL INFO Assigned NET plugin IB to comm +gpu1:238:238 [2] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:238:238 [2] NCCL INFO Using network IB +gpu1:242:242 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:242:242 [6] NCCL INFO Initialized NET plugin IB +gpu1:242:242 [6] NCCL INFO Assigned NET plugin IB to comm +gpu1:242:242 [6] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:242:242 [6] NCCL INFO Using network IB +gpu1:238:238 [2] NCCL INFO DMA-BUF is available on GPU device 2 +gpu1:238:238 [2] NCCL INFO ncclCommInitRank comm 0x219ba3b0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x61e0ecdb3f010655 - Init START +gpu1:239:239 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:239:239 [3] NCCL INFO Initialized NET plugin IB +gpu1:239:239 [3] NCCL INFO Assigned NET plugin IB to comm +gpu1:239:239 [3] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:239:239 [3] NCCL INFO Using network IB +gpu1:242:242 [6] NCCL INFO DMA-BUF is available on GPU device 6 +gpu1:242:242 [6] NCCL INFO ncclCommInitRank comm 0x2540ac80 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x61e0ecdb3f010655 - Init START +gpu1:239:239 [3] NCCL INFO DMA-BUF is available on GPU device 3 +gpu1:239:239 [3] NCCL INFO ncclCommInitRank comm 0x2c06d150 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x61e0ecdb3f010655 - Init START +gpu1:238:238 [2] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:242:242 [6] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:243:243 [7] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:240:240 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:240:240 [4] NCCL INFO Initialized NET plugin IB +gpu1:240:240 [4] NCCL INFO Assigned NET plugin IB to comm +gpu1:240:240 [4] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:240:240 [4] NCCL INFO Using network IB +gpu1:240:240 [4] NCCL INFO DMA-BUF is available on GPU device 4 +gpu1:240:240 [4] NCCL INFO ncclCommInitRank comm 0x482f7e60 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x61e0ecdb3f010655 - Init START +gpu1:241:241 [5] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:239:239 [3] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:240:240 [4] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:236:236 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu1:236:236 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu1:236:236 [0] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu1:236:236 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.11<0> +gpu1:236:236 [0] NCCL INFO Initialized NET plugin IB +gpu1:236:236 [0] NCCL INFO Assigned NET plugin IB to comm +gpu1:236:236 [0] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu1:236:236 [0] NCCL INFO Using network IB +gpu1:236:236 [0] NCCL INFO DMA-BUF is available on GPU device 0 +gpu1:236:236 [0] NCCL INFO ncclCommInitRank comm 0x46d583c0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x61e0ecdb3f010655 - Init START +gpu1:237:237 [1] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:236:236 [0] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu1:237:237 [1] NCCL INFO Bootstrap timings total 0.049632 (create 0.000024, send 0.000088, recv 0.010487, ring 0.000558, delay 0.000001) +gpu1:238:238 [2] NCCL INFO Bootstrap timings total 0.039417 (create 0.000044, send 0.000192, recv 0.012734, ring 0.025887, delay 0.000002) +gpu1:239:239 [3] NCCL INFO Bootstrap timings total 0.026976 (create 0.000049, send 0.000168, recv 0.011406, ring 0.014714, delay 0.000002) +gpu1:241:241 [5] NCCL INFO Bootstrap timings total 0.056288 (create 0.000027, send 0.000096, recv 0.030242, ring 0.014939, delay 0.000001) +gpu1:236:236 [0] NCCL INFO Bootstrap timings total 0.001248 (create 0.000026, send 0.000098, recv 0.000285, ring 0.000545, delay 0.000001) +gpu1:237:237 [1] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:236:236 [0] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:241:241 [5] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:238:238 [2] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:239:239 [3] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:240:240 [4] NCCL INFO Bootstrap timings total 0.015860 (create 0.000062, send 0.000281, recv 0.000229, ring 0.014658, delay 0.000002) +gpu1:240:240 [4] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:243:243 [7] NCCL INFO Bootstrap timings total 0.060146 (create 0.000031, send 0.000123, recv 0.015824, ring 0.025541, delay 0.000001) +gpu1:243:243 [7] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:242:242 [6] NCCL INFO Bootstrap timings total 0.029217 (create 0.000059, send 0.000261, recv 0.000206, ring 0.025484, delay 0.000003) +gpu1:242:242 [6] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu1:243:243 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:243:243 [7] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:243:243 [7] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:236:236 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:236:236 [0] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:236:236 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:238:238 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:238:238 [2] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:238:238 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:237:237 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:237:237 [1] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:241:241 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:241:241 [5] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:237:237 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:241:241 [5] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:242:242 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:242:242 [6] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:242:242 [6] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:240:240 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu1:240:240 [4] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:240:240 [4] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:239:239 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu1:239:239 [3] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu1:239:239 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu1:236:236 [0] NCCL INFO comm 0x46d583c0 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:237:237 [1] NCCL INFO comm 0x3a507100 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:238:238 [2] NCCL INFO comm 0x219ba3b0 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:239:239 [3] NCCL INFO comm 0x2c06d150 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu1:239:239 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 1/11/-1->3->-1 [2] -1/-1/-1->3->2 [3] 1/-1/-1->3->7 +gpu1:236:236 [0] NCCL INFO Channel 00/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:237:237 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->3 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->3 +gpu1:241:241 [5] NCCL INFO comm 0x4e055680 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:236:236 [0] NCCL INFO Channel 01/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:239:239 [3] NCCL INFO P2P Chunksize set to 131072 +gpu1:240:240 [4] NCCL INFO comm 0x482f7e60 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:238:238 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 0/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 0/-1/-1->2->1 +gpu1:237:237 [1] NCCL INFO P2P Chunksize set to 131072 +gpu1:236:236 [0] NCCL INFO Channel 02/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 03/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:238:238 [2] NCCL INFO P2P Chunksize set to 131072 +gpu1:241:241 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->7 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->7 +gpu1:241:241 [5] NCCL INFO P2P Chunksize set to 131072 +gpu1:236:236 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->2 [2] 1/-1/-1->0->4 [3] -1/-1/-1->0->2 +gpu1:236:236 [0] NCCL INFO P2P Chunksize set to 131072 +gpu1:240:240 [4] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->6 [2] 5/8/0->4->12 [3] -1/-1/-1->4->6 +gpu1:240:240 [4] NCCL INFO P2P Chunksize set to 131072 +gpu1:243:243 [7] NCCL INFO comm 0x1a399720 rank 7 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu1:242:242 [6] NCCL INFO comm 0x2540ac80 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:243:243 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 5/-1/-1->7->11 [2] -1/-1/-1->7->6 [3] 5/11/3->7->15 +gpu1:243:243 [7] NCCL INFO P2P Chunksize set to 131072 +gpu1:242:242 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 4/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 4/-1/-1->6->5 +gpu1:242:242 [6] NCCL INFO P2P Chunksize set to 131072 +gpu1:241:241 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:241:241 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:243:243 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:239:239 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:243:243 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:239:239 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:240:240 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:240:240 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:242:242 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:242:242 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:1587 [0] NCCL INFO [Proxy Service] Device 5 CPU core 35 +gpu1:236:236 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:236:236 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:237:237 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:237:237 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:238:238 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu1:238:238 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:1588 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 62 +gpu1:240:1592 [0] NCCL INFO [Proxy Service] Device 4 CPU core 44 +gpu1:239:1590 [0] NCCL INFO [Proxy Service] Device 3 CPU core 68 +gpu1:243:1589 [0] NCCL INFO [Proxy Service] Device 7 CPU core 54 +gpu1:243:1591 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 63 +gpu1:239:1594 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 69 +gpu1:240:1595 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 33 +gpu1:242:1593 [0] NCCL INFO [Proxy Service] Device 6 CPU core 58 +gpu1:236:1596 [0] NCCL INFO [Proxy Service] Device 0 CPU core 7 +gpu1:242:1597 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 46 +gpu1:237:1598 [0] NCCL INFO [Proxy Service] Device 1 CPU core 80 +gpu1:236:1599 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 73 +gpu1:238:1600 [0] NCCL INFO [Proxy Service] Device 2 CPU core 12 +gpu1:237:1601 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 17 +gpu1:238:1602 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 77 +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:240:1603 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 103 +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:1604 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 75 +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:243:1605 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 47 +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:1606 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 18 +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:242:242 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:241:241 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:240:240 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:237:237 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:238:238 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [send] via NET/IB/1/GDRDMA +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [send] via NET/IB/1/GDRDMA +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Connected all trees +gpu1:236:236 [0] NCCL INFO Connected all trees +gpu1:243:243 [7] NCCL INFO Connected all trees +gpu1:240:240 [4] NCCL INFO Connected all trees +gpu1:237:237 [1] NCCL INFO Connected all trees +gpu1:238:238 [2] NCCL INFO Connected all trees +gpu1:241:241 [5] NCCL INFO Connected all trees +gpu1:242:242 [6] NCCL INFO Connected all trees +gpu1:239:239 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:240:240 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:243:243 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:239:239 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:239:239 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:240:240 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:243:243 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:240:240 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:243:243 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:236:236 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:236:236 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO CC Off, workFifoBytes 1048576 +gpu1:242:1608 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 98 +gpu1:241:1607 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 38 +gpu1:241:241 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:241:241 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:241:241 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:237:1609 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 19 +gpu1:238:1610 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 20 +gpu1:242:242 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:242:242 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:242:242 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:237:237 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:237:237 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:237:237 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:238:238 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu1:238:238 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:238:238 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:242:242 [6] NCCL INFO ncclCommInitRank comm 0x2540ac80 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:242:242 [6] NCCL INFO Init timings - ncclCommInitRank: rank 6 nranks 16 total 0.67 (kernels 0.33, alloc 0.05, bootstrap 0.03, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.22, rest 0.00) +gpu1:236:236 [0] NCCL INFO ncclCommInitRank comm 0x46d583c0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:238:238 [2] NCCL INFO ncclCommInitRank comm 0x219ba3b0 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:236:236 [0] NCCL INFO Init timings - ncclCommInitRank: rank 0 nranks 16 total 0.67 (kernels 0.39, alloc 0.01, bootstrap 0.00, allgathers 0.01, topo 0.03, graphs 0.01, connections 0.17, rest 0.05) +gpu1:238:238 [2] NCCL INFO Init timings - ncclCommInitRank: rank 2 nranks 16 total 0.67 (kernels 0.32, alloc 0.04, bootstrap 0.04, allgathers 0.00, topo 0.03, graphs 0.01, connections 0.22, rest 0.00) +gpu1:237:237 [1] NCCL INFO ncclCommInitRank comm 0x3a507100 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:237:237 [1] NCCL INFO Init timings - ncclCommInitRank: rank 1 nranks 16 total 0.67 (kernels 0.34, alloc 0.01, bootstrap 0.05, allgathers 0.00, topo 0.04, graphs 0.01, connections 0.22, rest 0.00) +gpu1:239:239 [3] NCCL INFO ncclCommInitRank comm 0x2c06d150 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:239:239 [3] NCCL INFO Init timings - ncclCommInitRank: rank 3 nranks 16 total 0.65 (kernels 0.32, alloc 0.04, bootstrap 0.03, allgathers 0.00, topo 0.04, graphs 0.01, connections 0.17, rest 0.05) +gpu1:240:240 [4] NCCL INFO ncclCommInitRank comm 0x482f7e60 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:240:240 [4] NCCL INFO Init timings - ncclCommInitRank: rank 4 nranks 16 total 0.67 (kernels 0.32, alloc 0.07, bootstrap 0.02, allgathers 0.00, topo 0.04, graphs 0.00, connections 0.17, rest 0.05) +gpu1:241:241 [5] NCCL INFO ncclCommInitRank comm 0x4e055680 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:243:243 [7] NCCL INFO ncclCommInitRank comm 0x1a399720 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu1:241:241 [5] NCCL INFO Init timings - ncclCommInitRank: rank 5 nranks 16 total 0.67 (kernels 0.34, alloc 0.01, bootstrap 0.06, allgathers 0.01, topo 0.04, graphs 0.00, connections 0.22, rest 0.00) +gpu1:243:243 [7] NCCL INFO Init timings - ncclCommInitRank: rank 7 nranks 16 total 0.66 (kernels 0.31, alloc 0.01, bootstrap 0.06, allgathers 0.01, topo 0.03, graphs 0.00, connections 0.17, rest 0.05) +[2026-08-01 05:01:11 TP0 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP5 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP1 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP4 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP3 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP2 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP6 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP7 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP0 EP0] Init torch distributed ends. elapsed=8.19 s, mem usage=0.42 GB +[2026-08-01 05:01:11 TP7 EP0] Init torch distributed ends. elapsed=7.47 s, mem usage=0.48 GB +[2026-08-01 05:01:11 TP4 EP0] Init torch distributed ends. elapsed=7.73 s, mem usage=0.46 GB +[2026-08-01 05:01:11 TP3 EP0] Init torch distributed ends. elapsed=7.41 s, mem usage=0.44 GB +[2026-08-01 05:01:11 TP5 EP0] Init torch distributed ends. elapsed=7.47 s, mem usage=0.41 GB +[2026-08-01 05:01:11 TP6 EP0] Init torch distributed ends. elapsed=7.26 s, mem usage=0.39 GB +[2026-08-01 05:01:11 TP2 EP0] Init torch distributed ends. elapsed=7.73 s, mem usage=0.39 GB +[2026-08-01 05:01:11 TP1 EP0] Init torch distributed ends. elapsed=7.35 s, mem usage=0.41 GB +[2026-08-01 05:01:14 TP3 EP0] Load weight begin. avail mem=82.07 GB +[2026-08-01 05:01:14 TP7 EP0] Load weight begin. avail mem=82.03 GB +[2026-08-01 05:01:14 TP6 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:01:14 TP2 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:01:14 TP5 EP0] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:01:14 TP0 EP0] Load weight begin. avail mem=82.08 GB +[2026-08-01 05:01:14 TP0 EP0] Detected fp8 checkpoint. +[2026-08-01 05:01:14 TP0 EP0] Config does not support fused shared expert(s). Shared experts fusion optimization is disabled. +[2026-08-01 05:01:14 TP1 EP0] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:01:14 TP4 EP0] Load weight begin. avail mem=82.04 GB +[2026-08-01 05:01:14 TP3 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP7 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP6 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP2 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP5 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP1 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP4 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:19 TP1 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP2 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP0 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP3 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP5 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP4 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP7 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP6 EP0] multimem all-gather disabled because the TP group spans across nodes. + Multi-thread loading shards: 0% Completed | 0/64 [00:007->6 [1] 5/-1/-1->7->11 [2] -1/-1/-1->7->6 [3] 5/11/3->7->15 +gpu1:242:242 [6] NCCL INFO comm 0x51286f50 rank 6 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:243:243 [7] NCCL INFO P2P Chunksize set to 131072 +gpu1:243:243 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:241 [5] NCCL INFO comm 0x74595de0 rank 5 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:242:242 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 4/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 4/-1/-1->6->5 +gpu1:242:242 [6] NCCL INFO P2P Chunksize set to 131072 +gpu1:242:242 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:241 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->7 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->7 +gpu1:241:241 [5] NCCL INFO P2P Chunksize set to 131072 +gpu1:240:240 [4] NCCL INFO comm 0x6bf39aa0 rank 4 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:241:241 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:239:239 [3] NCCL INFO comm 0x5663d330 rank 3 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu1:238:238 [2] NCCL INFO comm 0x5051b460 rank 2 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu1:240:240 [4] NCCL INFO Trees [0] 5/-1/-1->4->8 [1] -1/-1/-1->4->6 [2] 5/8/0->4->12 [3] -1/-1/-1->4->6 +gpu1:239:239 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] 1/11/-1->3->-1 [2] -1/-1/-1->3->2 [3] 1/-1/-1->3->7 +gpu1:240:240 [4] NCCL INFO P2P Chunksize set to 131072 +gpu1:237:237 [1] NCCL INFO comm 0x62d36ef0 rank 1 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu1:239:239 [3] NCCL INFO P2P Chunksize set to 131072 +gpu1:239:239 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:240:240 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:238:238 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 0/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 0/-1/-1->2->1 +gpu1:236:236 [0] NCCL INFO comm 0x6e4f63a0 rank 0 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu1:238:238 [2] NCCL INFO P2P Chunksize set to 131072 +gpu1:237:237 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->3 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->3 +gpu1:238:238 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:237:237 [1] NCCL INFO P2P Chunksize set to 131072 +gpu1:237:237 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:236:236 [0] NCCL INFO Channel 00/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:243:2380 [0] NCCL INFO [Proxy Service] Device 7 CPU core 53 +gpu1:236:236 [0] NCCL INFO Channel 01/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 02/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:236:236 [0] NCCL INFO Channel 03/04 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +gpu1:241:2381 [0] NCCL INFO [Proxy Service] Device 5 CPU core 35 +gpu1:236:236 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->2 [2] 1/-1/-1->0->4 [3] -1/-1/-1->0->2 +gpu1:236:236 [0] NCCL INFO P2P Chunksize set to 131072 +gpu1:243:2382 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 56 +gpu1:236:236 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu1:241:2383 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 100 +gpu1:239:2384 [0] NCCL INFO [Proxy Service] Device 3 CPU core 14 +gpu1:239:2387 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 16 +gpu1:242:2389 [0] NCCL INFO [Proxy Service] Device 6 CPU core 102 +gpu1:240:2385 [0] NCCL INFO [Proxy Service] Device 4 CPU core 101 +gpu1:237:2388 [0] NCCL INFO [Proxy Service] Device 1 CPU core 65 +gpu1:242:2390 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 39 +gpu1:240:2391 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 40 +gpu1:237:2392 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 2 +gpu1:238:2386 [0] NCCL INFO [Proxy Service] Device 2 CPU core 5 +gpu1:238:2393 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 71 +gpu1:236:2394 [0] NCCL INFO [Proxy Service] Device 0 CPU core 72 +gpu1:236:2395 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 73 +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/IPC +gpu1:240:2396 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 105 +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:236:2397 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 74 +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC +gpu1:243:2398 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 57 +gpu1:239:2399 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 79 +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:238:238 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:239:239 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:237:237 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:236:236 [0] NCCL INFO Channel 01/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:241:241 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:243:243 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:237:237 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu1:240:240 [4] NCCL INFO Channel 01/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 01/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 03/0 : 0[0] -> 2[2] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 03/0 : 4[4] -> 6[6] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 03/0 : 5[5] -> 7[7] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 0[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 3[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 4[4] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [send] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:236:236 [0] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [receive] via NET/IB/0/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [send] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [receive] via NET/IB/0/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [receive] via NET/IB/1/GDRDMA +gpu1:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [receive] via NET/IB/1/GDRDMA +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 3[3] [send] via NET/IB/1/GDRDMA +gpu1:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 01/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 03/0 : 3[3] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 5[5] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 00/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:239:239 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 00/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:243:243 [7] NCCL INFO Channel 02/0 : 7[7] -> 6[6] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 01/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 02/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:242:242 [6] NCCL INFO Channel 03/0 : 6[6] -> 5[5] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 00/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:238:238 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC +gpu1:237:237 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC +gpu1:241:241 [5] NCCL INFO Channel 02/0 : 5[5] -> 4[4] via P2P/IPC +gpu1:240:240 [4] NCCL INFO Connected all trees +gpu1:241:241 [5] NCCL INFO Connected all trees +gpu1:243:243 [7] NCCL INFO Connected all trees +gpu1:242:242 [6] NCCL INFO Connected all trees +gpu1:239:239 [3] NCCL INFO Connected all trees +gpu1:240:240 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:240:240 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:243:243 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:243:243 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:239:239 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:239:239 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO Connected all trees +gpu1:238:238 [2] NCCL INFO Connected all trees +gpu1:237:237 [1] NCCL INFO Connected all trees +gpu1:236:236 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:236:236 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:236:236 [0] NCCL INFO CC Off, workFifoBytes 1048576 +gpu1:242:2400 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 106 +gpu1:241:2401 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 107 +gpu1:241:241 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:241:241 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:242:242 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:242:242 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:238:2402 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 77 +gpu1:242:242 [6] NCCL INFO ncclCommInitRankConfig comm 0x51286f50 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:242:242 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 16 total 0.31 (kernels 0.00, alloc 0.00, bootstrap 0.03, allgathers 0.00, topo 0.06, graphs 0.00, connections 0.21, rest 0.00) +gpu1:240:240 [4] NCCL INFO ncclCommInitRankConfig comm 0x6bf39aa0 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:240:240 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 16 total 0.29 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.00, topo 0.05, graphs 0.01, connections 0.15, rest 0.05) +gpu1:238:238 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:238:238 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:237:2403 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 81 +gpu1:237:237 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu1:237:237 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu1:241:241 [5] NCCL INFO ncclCommInitRankConfig comm 0x74595de0 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:241:241 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 16 total 0.28 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.00, topo 0.06, graphs 0.00, connections 0.21, rest 0.00) +gpu1:243:243 [7] NCCL INFO ncclCommInitRankConfig comm 0x45ca6780 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:243:243 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 16 total 0.29 (kernels 0.00, alloc 0.00, bootstrap 0.02, allgathers 0.00, topo 0.06, graphs 0.00, connections 0.16, rest 0.05) +gpu1:238:238 [2] NCCL INFO ncclCommInitRankConfig comm 0x5051b460 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:238:238 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 16 total 0.28 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.06, graphs 0.01, connections 0.21, rest 0.00) +gpu1:236:236 [0] NCCL INFO ncclCommInitRankConfig comm 0x6e4f63a0 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:236:236 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 16 total 0.36 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.00, topo 0.06, graphs 0.01, connections 0.16, rest 0.06) +gpu1:237:237 [1] NCCL INFO ncclCommInitRankConfig comm 0x62d36ef0 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:237:237 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 16 total 0.29 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.00, topo 0.06, graphs 0.01, connections 0.21, rest 0.00) +gpu1:239:239 [3] NCCL INFO ncclCommInitRankConfig comm 0x5663d330 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu1:239:239 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 16 total 0.29 (kernels 0.00, alloc 0.00, bootstrap 0.01, allgathers 0.01, topo 0.05, graphs 0.00, connections 0.16, rest 0.06) +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-08-01 05:04:01 TP3 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP3 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP3 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.77 GB +[2026-08-01 05:04:01 TP5 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP5 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP4 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP4 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP2 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP7 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP2 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP7 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP0 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP0 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP1 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP1 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP5 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:04:01 TP4 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.72 GB +[2026-08-01 05:04:01 TP2 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:04:01 TP7 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.69 GB +[2026-08-01 05:04:01 TP1 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:04:01 TP0 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.80 GB +[2026-08-01 05:04:01 TP6 EP0] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP6 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP6 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB + 0%| | 0/12 [00:00, ) +[2026-08-01 05:05:31 TP3 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP6 EP0] Capture target decode CUDA graph end. elapsed=89.85 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:05:31 TP6 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP6 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP5 EP0] Capture target decode CUDA graph end. elapsed=89.92 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:05:31 TP5 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP5 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP2 EP0] Capture target decode CUDA graph end. elapsed=89.92 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:05:31 TP2 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP2 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP7 EP0] Capture target decode CUDA graph end. elapsed=89.92 s, mem usage=1.37 GB, avail mem=5.32 GB. +[2026-08-01 05:05:31 TP7 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP7 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP1 EP0] Capture target decode CUDA graph end. elapsed=89.93 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:05:31 TP4 EP0] Capture target decode CUDA graph end. elapsed=89.93 s, mem usage=1.37 GB, avail mem=5.35 GB. +[2026-08-01 05:05:31 TP1 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP1 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP4 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP4 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP0 EP0] Capture target decode CUDA graph end. elapsed=89.96 s, mem usage=1.37 GB, avail mem=5.43 GB. +[2026-08-01 05:05:31 TP0 EP0] max_total_num_tokens=1270016, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=256, context_len=1048576, available_gpu_mem=5.43 GB +[2026-08-01 05:05:31 TP0 EP0] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP0 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:32] INFO: Started server process [1] +[2026-08-01 05:05:32] INFO: Waiting for application startup. +[2026-08-01 05:05:32] Using default chat sampling params from model generation config: {'temperature': 1.0, 'top_p': 1.0} +[2026-08-01 05:05:32] INFO: Application startup complete. +[2026-08-01 05:05:32] INFO: Uvicorn running on http://0.0.0.0:30002 (Press CTRL+C to quit) +[2026-08-01 05:05:33] INFO: 127.0.0.1:42962 - "GET /model_info HTTP/1.1" 200 OK +[2026-08-01 05:05:33] INFO: 10.101.0.11:57250 - "GET /health HTTP/1.1" 503 Service Unavailable +[2026-08-01 05:05:37 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 45.26 +[2026-08-01 05:05:37] INFO: 127.0.0.1:42974 - "POST /generate HTTP/1.1" 200 OK +[2026-08-01 05:05:37] The server is fired up and ready to roll! +[2026-08-01 05:05:39 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 108.63 +[2026-08-01 05:05:39] INFO: 10.101.0.11:60750 - "GET /health HTTP/1.1" 200 OK diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_server_cmd.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_server_cmd.txt new file mode 100644 index 0000000..ff6de9b --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/head_server_cmd.txt @@ -0,0 +1 @@ +docker run -d --name dsv4pro_pro6000d_2node_sglang_tp16_quick_map_head --gpus all --network host --ipc host --shm-size 20g --ulimit memlock=-1 --ulimit stack=67108864 -v /data/hf_models/DeepSeek-V4-Pro:/data/hf_models/DeepSeek-V4-Pro:ro -v /data/hzy/sglang_cache/dsv4_pro_tp16:/root/.cache -e CUDA_VISIBLE_DEVICES=0\,1\,2\,3\,4\,5\,6\,7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -e NCCL_SOCKET_IFNAME=eth0 -e NCCL_IB_HCA==mlx5_0:1\,mlx5_3:1 -e NCCL_CROSS_NIC=1 -e NCCL_DEBUG=INFO -e SGLANG_SHARED_EXPERT_TP1=1 --device /dev/infiniband/rdma_cm --device /dev/infiniband/uverbs0 --device /dev/infiniband/uverbs3 --entrypoint python3 lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45 -m sglang.launch_server --model-path /data/hf_models/DeepSeek-V4-Pro --tp-size 16 --ep-size 2 --nnodes 2 --node-rank 0 --dist-init-addr 10.101.0.11:20002 --trust-remote-code --host 0.0.0.0 --port 30002 --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_nccl_transport.log b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_nccl_transport.log new file mode 100644 index 0000000..c64c1fc --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_nccl_transport.log @@ -0,0 +1,918 @@ +/usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). +/sgl-workspace/sglang/python/sglang/launch_server.py:56: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-01 05:00:37] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:37] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:37] Breakable CUDA graph is incompatible with DeepSeek-V4 (heavy capture-pool memory pressure); disabling prefill CUDA graph. +[2026-08-01 05:00:37] Use dsv4 attention backend for DeepseekV4ForCausalLM, setting page_size to 256. +[2026-08-01 05:00:37] Setting swa_full_tokens_ratio to 0.1 for DeepseekV4ForCausalLM. +[2026-08-01 05:00:37] Setting KV cache dtype to fp8_e4m3 for DeepseekV4ForCausalLM. +[2026-08-01 05:00:37] Use flashinfer_mxfp4 as MoE runner backend on SM120 for DeepseekV4 +[2026-08-01 05:00:40] server_args=ServerArgs(model_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_path='/data/hf_models/DeepSeek-V4-Pro', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=None, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.9, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=8192, enable_dynamic_chunking=False, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, retraction_policy='length', schedule_conservativeness=1.0, page_size=256, swa_full_tokens_ratio=0.1, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, enable_page_major_kv_layout=False, enable_unified_memory=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, nccl_port=None, dist_timeout=None, dist_init_addr='10.101.0.11:20002', nnodes=2, node_rank=1, tp_size=16, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=1, load_balance_method='round_robin', attn_cp_size=1, moe_dp_size=1, enable_prefill_cp=False, cp_strategy=None, enable_dsa_cache_layer_split=False, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_dp_attention=False, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=False, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=434986221, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, host='0.0.0.0', port=30002, fastapi_root_path='', smg_grpc_mode=False, grpc_mode=False, grpc_port=None, skip_server_warmup=False, warmups=None, enable_http2=False, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, api_key=None, admin_api_key=None, served_model_name='/data/hf_models/DeepSeek-V4-Pro', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, default_chat_template_kwargs=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser=None, tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, smg_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, attention_backend='dsv4', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='auto', bf16_gemm_backend='auto', dsa_prefill_backend=None, dsa_decode_backend=None, dsa_paged_mqa_logits_backend='auto', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, mamba_backend='triton', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=64, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], tc_compiler='eager', full_prefill_max_req=None), prefill=PhaseConfig(backend='disabled', max_bs=8192, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], tc_compiler='eager', full_prefill_max_req=None)), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=64, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, torchao_config='', speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_dspark_block_size=None, speculative_dspark_sps_table_path=None, speculative_dspark_confidence_sts_path=None, speculative_dspark_align_verify_tokens_to_graph_tier=False, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_window_size=None, speculative_moe_runner_backend='flashinfer_mxfp4', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=2, moe_a2a_backend='none', moe_runner_backend='flashinfer_mxfp4', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', fuseep_mode=2, deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_waterfill=False, ep_join_mode=None, ep_join_rank_offset=0, elastic_ep_initial_size=None, max_ep_size=None, elastic_ep_scale_timeout=600, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=False, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, enable_mamba_cache_stochastic_rounding=False, mamba_cache_philox_rounds=0, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', uses_mamba_radix_cache=False, mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, enable_mm_global_cache=False, disable_fast_image_processor=False, mm_feature_transport='cpu', keep_mm_feature_on_device=False, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', enable_lmcache=False, lmcache_config_file=None, enable_flexkv=False, flexkv_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, dllm_fdfo=True, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_attempts=0, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, forward_hooks=None, msprobe_dump_config=None) +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:40] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-01 05:00:53 TP12 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP12 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP10 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP10 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP15 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP15 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP14 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP14 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP13 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP13 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP9 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP9 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP11 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP11 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:53 TP8 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:53 TP8 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:55 TP12 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:55 TP12 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:55 TP12 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:55 TP15 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:55 TP10 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:55 TP12 EP1] Init torch distributed begin. +[2026-08-01 05:00:55 TP14 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:56 TP8 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:56 TP13 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:56 TP9 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:56 TP15 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP15 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP11 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:00:56 TP10 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP10 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP15 EP1] Init torch distributed begin. +[2026-08-01 05:00:56 TP10 EP1] Init torch distributed begin. +[2026-08-01 05:00:56 TP14 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP14 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP8 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP8 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP13 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP13 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP9 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP9 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP14 EP1] Init torch distributed begin. +[2026-08-01 05:00:56 TP8 EP1] Init torch distributed begin. +[2026-08-01 05:00:56 TP13 EP1] Init torch distributed begin. +[2026-08-01 05:00:56 TP11 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True +[2026-08-01 05:00:56 TP11 EP1] Hybrid swa model: self.hf_config.architectures=['DeepseekV4ForCausalLM'] +[2026-08-01 05:00:56 TP9 EP1] Init torch distributed begin. +[2026-08-01 05:00:56 TP11 EP1] Init torch distributed begin. +gpu3:241:241 [5] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:241:241 [5] NCCL INFO cudaDriverVersion 13000 +gpu3:241:241 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:241:241 [5] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:241:241 [5] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:236:236 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:236:236 [0] NCCL INFO cudaDriverVersion 13000 +gpu3:237:237 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:237:237 [1] NCCL INFO cudaDriverVersion 13000 +gpu3:236:236 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:236:236 [0] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:236:236 [0] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:237:237 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [6] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:242:242 [6] NCCL INFO cudaDriverVersion 13000 +gpu3:237:237 [1] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:237:237 [1] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:238:238 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:238:238 [2] NCCL INFO cudaDriverVersion 13000 +gpu3:240:240 [4] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:240:240 [4] NCCL INFO cudaDriverVersion 13000 +gpu3:239:239 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:239:239 [3] NCCL INFO cudaDriverVersion 13000 +gpu3:242:242 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:240:240 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:238:238 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:239:239 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [6] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:242:242 [6] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:240:240 [4] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:240:240 [4] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:239:239 [3] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:238:238 [2] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:239:239 [3] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:238:238 [2] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:243:243 [7] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so +gpu3:243:243 [7] NCCL INFO cudaDriverVersion 13000 +gpu3:243:243 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:243:243 [7] NCCL INFO Bootstrap: Using eth0:10.101.0.13<0> +gpu3:243:243 [7] NCCL INFO NCCL version 2.28.9+cuda13.0 +gpu3:241:241 [5] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:241:241 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:241:241 [5] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:242:242 [6] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:242:242 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:242:242 [6] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:241:241 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:241:241 [5] NCCL INFO Initialized NET plugin IB +gpu3:241:241 [5] NCCL INFO Assigned NET plugin IB to comm +gpu3:241:241 [5] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:241:241 [5] NCCL INFO Using network IB +gpu3:240:240 [4] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:240:240 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:240:240 [4] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:241:241 [5] NCCL INFO DMA-BUF is available on GPU device 5 +gpu3:241:241 [5] NCCL INFO ncclCommInitRank comm 0x17d21f10 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x61e0ecdb3f010655 - Init START +gpu3:242:242 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:242:242 [6] NCCL INFO Initialized NET plugin IB +gpu3:242:242 [6] NCCL INFO Assigned NET plugin IB to comm +gpu3:242:242 [6] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:242:242 [6] NCCL INFO Using network IB +gpu3:240:240 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:240:240 [4] NCCL INFO Initialized NET plugin IB +gpu3:240:240 [4] NCCL INFO Assigned NET plugin IB to comm +gpu3:240:240 [4] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:240:240 [4] NCCL INFO Using network IB +gpu3:242:242 [6] NCCL INFO DMA-BUF is available on GPU device 6 +gpu3:242:242 [6] NCCL INFO ncclCommInitRank comm 0x3ce594c0 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x61e0ecdb3f010655 - Init START +gpu3:240:240 [4] NCCL INFO DMA-BUF is available on GPU device 4 +gpu3:240:240 [4] NCCL INFO ncclCommInitRank comm 0x30394420 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x61e0ecdb3f010655 - Init START +gpu3:241:241 [5] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:243:243 [7] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:243:243 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:243:243 [7] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:239:239 [3] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:239:239 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:239:239 [3] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:239:239 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:239:239 [3] NCCL INFO Initialized NET plugin IB +gpu3:239:239 [3] NCCL INFO Assigned NET plugin IB to comm +gpu3:239:239 [3] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:239:239 [3] NCCL INFO Using network IB +gpu3:238:238 [2] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:238:238 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:238:238 [2] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:239:239 [3] NCCL INFO DMA-BUF is available on GPU device 3 +gpu3:239:239 [3] NCCL INFO ncclCommInitRank comm 0x1514e8b0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x61e0ecdb3f010655 - Init START +gpu3:240:240 [4] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:236:236 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:236:236 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:236:236 [0] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:243:243 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:243:243 [7] NCCL INFO Initialized NET plugin IB +gpu3:243:243 [7] NCCL INFO Assigned NET plugin IB to comm +gpu3:243:243 [7] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:243:243 [7] NCCL INFO Using network IB +gpu3:237:237 [1] NCCL INFO NET/Plugin: Could not find: libnccl-net.so +gpu3:237:237 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 +gpu3:237:237 [1] NCCL INFO NCCL_IB_HCA set to =mlx5_0:1,mlx5_3:1 +gpu3:243:243 [7] NCCL INFO DMA-BUF is available on GPU device 7 +gpu3:243:243 [7] NCCL INFO ncclCommInitRank comm 0x38edd050 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x61e0ecdb3f010655 - Init START +gpu3:242:242 [6] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:237:237 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:237:237 [1] NCCL INFO Initialized NET plugin IB +gpu3:237:237 [1] NCCL INFO Assigned NET plugin IB to comm +gpu3:237:237 [1] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:237:237 [1] NCCL INFO Using network IB +gpu3:238:238 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:238:238 [2] NCCL INFO Initialized NET plugin IB +gpu3:238:238 [2] NCCL INFO Assigned NET plugin IB to comm +gpu3:238:238 [2] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:238:238 [2] NCCL INFO Using network IB +gpu3:237:237 [1] NCCL INFO DMA-BUF is available on GPU device 1 +gpu3:237:237 [1] NCCL INFO ncclCommInitRank comm 0x270853b0 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x61e0ecdb3f010655 - Init START +gpu3:238:238 [2] NCCL INFO DMA-BUF is available on GPU device 2 +gpu3:238:238 [2] NCCL INFO ncclCommInitRank comm 0x20695320 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x61e0ecdb3f010655 - Init START +gpu3:238:238 [2] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:239:239 [3] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:236:236 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_3:1/RoCE [RO]; OOB eth0:10.101.0.13<0> +gpu3:236:236 [0] NCCL INFO Initialized NET plugin IB +gpu3:236:236 [0] NCCL INFO Assigned NET plugin IB to comm +gpu3:236:236 [0] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:236:236 [0] NCCL INFO Using network IB +gpu3:236:236 [0] NCCL INFO DMA-BUF is available on GPU device 0 +gpu3:236:236 [0] NCCL INFO ncclCommInitRank comm 0x27f51910 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x61e0ecdb3f010655 - Init START +gpu3:236:236 [0] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:237:237 [1] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:243:243 [7] NCCL INFO RAS client listening socket at 127.0.0.1<28028> +gpu3:241:241 [5] NCCL INFO Bootstrap timings total 0.142223 (create 0.000032, send 0.000175, recv 0.012391, ring 0.128299, delay 0.000001) +gpu3:243:243 [7] NCCL INFO Bootstrap timings total 0.085450 (create 0.000024, send 0.000145, recv 0.084412, ring 0.000569, delay 0.000001) +gpu3:241:241 [5] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:243:243 [7] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:242:242 [6] NCCL INFO Bootstrap timings total 0.130144 (create 0.000029, send 0.000132, recv 0.045068, ring 0.084588, delay 0.000001) +gpu3:242:242 [6] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:240:240 [4] NCCL INFO Bootstrap timings total 0.129069 (create 0.000025, send 0.000150, recv 0.000251, ring 0.103885, delay 0.000001) +gpu3:240:240 [4] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:239:239 [3] NCCL INFO Bootstrap timings total 0.104729 (create 0.000025, send 0.000157, recv 0.000296, ring 0.068743, delay 0.000001) +gpu3:239:239 [3] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:238:238 [2] NCCL INFO Bootstrap timings total 0.069643 (create 0.000024, send 0.000104, recv 0.000458, ring 0.068806, delay 0.000001) +gpu3:238:238 [2] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:236:236 [0] NCCL INFO Bootstrap timings total 0.044603 (create 0.000028, send 0.000167, recv 0.000479, ring 0.043650, delay 0.000001) +gpu3:237:237 [1] NCCL INFO Bootstrap timings total 0.070043 (create 0.000031, send 0.000139, recv 0.000614, ring 0.043580, delay 0.000001) +gpu3:236:236 [0] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:237:237 [1] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0. +gpu3:240:240 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:240:240 [4] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:240:240 [4] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:241:241 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:241:241 [5] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:241:241 [5] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:239:239 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:239:239 [3] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:242:242 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:239:239 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:242:242 [6] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:242:242 [6] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:238:238 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:238:238 [2] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:238:238 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:243:243 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:243:243 [7] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:243:243 [7] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:237:237 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:237:237 [1] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:237:237 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:236:236 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:236:236 [0] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0. +gpu3:236:236 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpu3:243:243 [7] NCCL INFO comm 0x38edd050 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:242:242 [6] NCCL INFO comm 0x3ce594c0 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:243:243 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 13/-1/-1->15->11 [2] -1/-1/-1->15->14 [3] 13/7/-1->15->-1 +gpu3:243:243 [7] NCCL INFO P2P Chunksize set to 131072 +gpu3:242:242 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 12/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 12/-1/-1->14->13 +gpu3:242:242 [6] NCCL INFO P2P Chunksize set to 131072 +gpu3:241:241 [5] NCCL INFO comm 0x17d21f10 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:241:241 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->15 [2] 14/-1/-1->13->12 [3] 14/-1/-1->13->15 +gpu3:241:241 [5] NCCL INFO P2P Chunksize set to 131072 +gpu3:240:240 [4] NCCL INFO comm 0x30394420 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:240:240 [4] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->14 [2] 13/4/-1->12->-1 [3] -1/-1/-1->12->14 +gpu3:240:240 [4] NCCL INFO P2P Chunksize set to 131072 +gpu3:239:239 [3] NCCL INFO comm 0x1514e8b0 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:238:238 [2] NCCL INFO comm 0x20695320 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:239:239 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 9/7/15->11->3 [2] -1/-1/-1->11->10 [3] 9/-1/-1->11->7 +gpu3:239:239 [3] NCCL INFO P2P Chunksize set to 131072 +gpu3:238:238 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 8/-1/-1->10->9 [2] 11/-1/-1->10->9 [3] 8/-1/-1->10->9 +gpu3:242:242 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:238:238 [2] NCCL INFO P2P Chunksize set to 131072 +gpu3:243:243 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:242:242 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:243:243 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:241:241 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:241:241 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:236:236 [0] NCCL INFO comm 0x27f51910 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:237:237 [1] NCCL INFO comm 0x270853b0 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:240:240 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:236:236 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->10 [2] 9/-1/-1->8->4 [3] -1/-1/-1->8->10 +gpu3:240:240 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:236:236 [0] NCCL INFO P2P Chunksize set to 131072 +gpu3:237:237 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/-1/-1->9->11 [2] 10/-1/-1->9->8 [3] 10/-1/-1->9->11 +gpu3:237:237 [1] NCCL INFO P2P Chunksize set to 131072 +gpu3:243:1437 [0] NCCL INFO [Proxy Service] Device 7 CPU core 125 +gpu3:241:1438 [0] NCCL INFO [Proxy Service] Device 5 CPU core 32 +gpu3:241:1441 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 110 +gpu3:239:239 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:239:239 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:236:236 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:236:236 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:238:238 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:238:238 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:243:1440 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 121 +gpu3:242:1436 [0] NCCL INFO [Proxy Service] Device 6 CPU core 99 +gpu3:240:1443 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 109 +gpu3:237:237 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so +gpu3:237:237 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:242:1439 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 38 +gpu3:240:1442 [0] NCCL INFO [Proxy Service] Device 4 CPU core 127 +gpu3:236:1445 [0] NCCL INFO [Proxy Service] Device 0 CPU core 66 +gpu3:239:1444 [0] NCCL INFO [Proxy Service] Device 3 CPU core 26 +gpu3:236:1448 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 8 +gpu3:238:1447 [0] NCCL INFO [Proxy Service] Device 2 CPU core 28 +gpu3:239:1446 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 27 +gpu3:237:1450 [0] NCCL INFO [Proxy Service] Device 1 CPU core 93 +gpu3:238:1449 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 25 +gpu3:237:1451 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 30 +gpu3:242:242 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 01/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 01/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 03/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 03/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 02/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 03/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:243:1452 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 126 +gpu3:243:243 [7] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:240:1453 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 39 +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 01/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 03/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:236:1454 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 73 +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 01/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:1455 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 31 +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:237:237 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:238:238 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:239:239 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:236:236 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:237:237 [1] NCCL INFO Channel 01/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 01/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:241:241 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:240:240 [4] NCCL INFO Channel 01/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 01/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 03/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 03/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 01/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:242:242 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [send] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:242:242 [6] NCCL INFO Channel 01/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [send] via NET/IB/1/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [send] via NET/IB/0/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:242:242 [6] NCCL INFO Channel 03/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [send] via NET/IB/1/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 00/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 02/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 02/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 00/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 02/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 00/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 02/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 03/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Connected all trees +gpu3:240:240 [4] NCCL INFO Connected all trees +gpu3:239:239 [3] NCCL INFO Connected all trees +gpu3:236:236 [0] NCCL INFO Connected all trees +gpu3:242:242 [6] NCCL INFO Connected all trees +gpu3:241:241 [5] NCCL INFO Connected all trees +gpu3:238:238 [2] NCCL INFO Connected all trees +gpu3:237:237 [1] NCCL INFO Connected all trees +gpu3:236:236 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:239:239 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:236:236 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:236:236 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:239:239 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:239:239 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:240:240 [4] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:240:240 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:240:240 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:243:243 [7] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:243:243 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:243:243 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:241:1456 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 122 +gpu3:241:241 [5] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:241:241 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:241:241 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:242:1457 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 101 +gpu3:242:242 [6] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:242:242 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:242:242 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:241:241 [5] NCCL INFO ncclCommInitRank comm 0x17d21f10 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:242:242 [6] NCCL INFO ncclCommInitRank comm 0x3ce594c0 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:243:243 [7] NCCL INFO ncclCommInitRank comm 0x38edd050 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:242:242 [6] NCCL INFO Init timings - ncclCommInitRank: rank 14 nranks 16 total 0.67 (kernels 0.25, alloc 0.02, bootstrap 0.13, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.22, rest 0.00) +gpu3:240:240 [4] NCCL INFO ncclCommInitRank comm 0x30394420 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:241:241 [5] NCCL INFO Init timings - ncclCommInitRank: rank 13 nranks 16 total 0.68 (kernels 0.23, alloc 0.03, bootstrap 0.14, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.19, rest 0.03) +gpu3:243:243 [7] NCCL INFO Init timings - ncclCommInitRank: rank 15 nranks 16 total 0.66 (kernels 0.27, alloc 0.04, bootstrap 0.09, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.17, rest 0.05) +gpu3:240:240 [4] NCCL INFO Init timings - ncclCommInitRank: rank 12 nranks 16 total 0.67 (kernels 0.25, alloc 0.02, bootstrap 0.13, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.17, rest 0.05) +gpu3:238:1458 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 68 +gpu3:237:1459 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 5 +gpu3:238:238 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:238:238 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:238:238 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:237:237 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so +gpu3:237:237 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:237:237 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:238:238 [2] NCCL INFO ncclCommInitRank comm 0x20695320 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:236:236 [0] NCCL INFO ncclCommInitRank comm 0x27f51910 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:238:238 [2] NCCL INFO Init timings - ncclCommInitRank: rank 10 nranks 16 total 0.67 (kernels 0.29, alloc 0.04, bootstrap 0.07, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.22, rest 0.00) +gpu3:236:236 [0] NCCL INFO Init timings - ncclCommInitRank: rank 8 nranks 16 total 0.67 (kernels 0.30, alloc 0.05, bootstrap 0.04, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.17, rest 0.05) +gpu3:239:239 [3] NCCL INFO ncclCommInitRank comm 0x1514e8b0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:237:237 [1] NCCL INFO ncclCommInitRank comm 0x270853b0 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x61e0ecdb3f010655 - Init COMPLETE +gpu3:239:239 [3] NCCL INFO Init timings - ncclCommInitRank: rank 11 nranks 16 total 0.67 (kernels 0.28, alloc 0.01, bootstrap 0.10, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.17, rest 0.05) +gpu3:237:237 [1] NCCL INFO Init timings - ncclCommInitRank: rank 9 nranks 16 total 0.67 (kernels 0.31, alloc 0.02, bootstrap 0.07, allgathers 0.02, topo 0.02, graphs 0.00, connections 0.22, rest 0.00) +[2026-08-01 05:01:11 TP15 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP14 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP13 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP12 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP10 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP9 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP8 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP11 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-01 05:01:11 TP15 EP1] Init torch distributed ends. elapsed=15.28 s, mem usage=0.44 GB +[2026-08-01 05:01:11 TP14 EP1] Init torch distributed ends. elapsed=15.10 s, mem usage=0.39 GB +[2026-08-01 05:01:11 TP13 EP1] Init torch distributed ends. elapsed=15.06 s, mem usage=0.41 GB +[2026-08-01 05:01:11 TP11 EP1] Init torch distributed ends. elapsed=14.94 s, mem usage=0.48 GB +[2026-08-01 05:01:11 TP12 EP1] Init torch distributed ends. elapsed=15.60 s, mem usage=0.42 GB +[2026-08-01 05:01:11 TP10 EP1] Init torch distributed ends. elapsed=15.26 s, mem usage=0.39 GB +[2026-08-01 05:01:11 TP8 EP1] Init torch distributed ends. elapsed=15.10 s, mem usage=0.46 GB +[2026-08-01 05:01:11 TP9 EP1] Init torch distributed ends. elapsed=14.99 s, mem usage=0.41 GB +[2026-08-01 05:01:14 TP14 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:01:14 TP8 EP1] Load weight begin. avail mem=82.04 GB +[2026-08-01 05:01:14 TP13 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:01:14 TP12 EP1] Load weight begin. avail mem=82.08 GB +[2026-08-01 05:01:14 TP15 EP1] Load weight begin. avail mem=82.07 GB +[2026-08-01 05:01:14 TP11 EP1] Load weight begin. avail mem=82.03 GB +[2026-08-01 05:01:14 TP10 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-01 05:01:14 TP9 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-01 05:01:14 TP13 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP8 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP14 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP12 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP15 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP11 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP9 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:14 TP10 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4FlashinferCutlassMoEMethod). +[2026-08-01 05:01:19 TP15 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP14 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP13 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP11 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP10 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP9 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP12 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:19 TP8 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-01 05:01:36 TP8 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:38 TP12 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:38 TP14 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:38 TP13 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:38 TP10 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:38 TP15 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:38 TP11 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:01:39 TP9 EP1] Execute dequant fp8 wo_a +[2026-08-01 05:02:59 TP8 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:02:59 TP8 EP1] Load weight end. elapsed=105.05 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.17 GB, mem usage=56.87 GB. +[2026-08-01 05:03:00 TP14 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:00 TP14 EP1] Load weight end. elapsed=106.16 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.24 GB, mem usage=56.87 GB. +[2026-08-01 05:03:00 TP10 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:00 TP10 EP1] Load weight end. elapsed=106.36 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.24 GB, mem usage=56.87 GB. +[2026-08-01 05:03:01 TP15 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:01 TP15 EP1] Load weight end. elapsed=106.51 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.19 GB, mem usage=56.87 GB. +[2026-08-01 05:03:01 TP12 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:01 TP12 EP1] Load weight end. elapsed=106.85 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.21 GB, mem usage=56.87 GB. +[2026-08-01 05:03:02 TP13 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:02 TP13 EP1] Load weight end. elapsed=107.72 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.23 GB, mem usage=56.87 GB. +[2026-08-01 05:03:02 TP9 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:02 TP9 EP1] Load weight end. elapsed=107.98 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.23 GB, mem usage=56.87 GB. +[2026-08-01 05:03:03 TP11 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! +[2026-08-01 05:03:03 TP11 EP1] Load weight end. elapsed=109.04 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=25.16 GB, mem usage=56.87 GB. +[2026-08-01 05:03:30 TP8 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP11 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP12 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP14 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP10 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP9 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP15 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:30 TP13 EP1] Tokenizer for /data/hf_models/DeepSeek-V4-Pro is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. +[2026-08-01 05:03:31 TP15 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP15 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP13 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP13 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP12 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP10 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP12 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP8 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP10 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP9 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP8 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP14 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP9 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP14 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP11 EP1] DSV4 memory calculation: bytes_per_full_token=10993.84, available_bytes=16.93 GB, c128_state_fixed=3.92 GB, full_token=1270016 +[2026-08-01 05:03:31 TP11 EP1] DSV4 pool sizes: full=1270016, swa=126976, c4=317504, c128=9922, c4_state=7936, c128_state=0 +[2026-08-01 05:03:31 TP10 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP11 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP9 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP8 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP14 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP13 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP12 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:31 TP15 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=256 swa_size=126976 c4_size=317504 c4_logical_size=317504 c128_size=9922 c4_state_pool_size=7936 c128_state_pool_size=32896 +[2026-08-01 05:03:32 TP11 EP1] Memory pool end. avail mem=7.09 GB +[2026-08-01 05:03:32 TP13 EP1] Memory pool end. avail mem=7.16 GB +[2026-08-01 05:03:32 TP12 EP1] Memory pool end. avail mem=7.14 GB +[2026-08-01 05:03:32 TP9 EP1] Memory pool end. avail mem=7.16 GB +[2026-08-01 05:03:32 TP10 EP1] Memory pool end. avail mem=7.18 GB +[2026-08-01 05:03:32 TP14 EP1] Memory pool end. avail mem=7.18 GB +[2026-08-01 05:03:32 TP15 EP1] Memory pool end. avail mem=7.13 GB +[2026-08-01 05:03:32 TP8 EP1] Memory pool end. avail mem=7.10 GB +[2026-08-01 05:03:32 TP11 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP9 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP13 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP12 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP10 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP11 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp11_pp0_dp0.json +[2026-08-01 05:03:32 TP14 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP9 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp9_pp0_dp0.json +[2026-08-01 05:03:32 TP13 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp13_pp0_dp0.json +[2026-08-01 05:03:32 TP12 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp12_pp0_dp0.json +[2026-08-01 05:03:32 TP10 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp10_pp0_dp0.json +[2026-08-01 05:03:32 TP15 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP8 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA). +[2026-08-01 05:03:32 TP14 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp14_pp0_dp0.json +[2026-08-01 05:03:32 TP15 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp15_pp0_dp0.json +[2026-08-01 05:03:32 TP8 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.14/sm120/4f24e70399961c55/rank_tp8_pp0_dp0.json +gpu3:242:242 [6] NCCL INFO Comm config Blocking set to 1 +gpu3:236:236 [0] NCCL INFO Comm config Blocking set to 1 +gpu3:238:238 [2] NCCL INFO Comm config Blocking set to 1 +gpu3:243:243 [7] NCCL INFO Comm config Blocking set to 1 +gpu3:237:237 [1] NCCL INFO Comm config Blocking set to 1 +gpu3:241:241 [5] NCCL INFO Comm config Blocking set to 1 +gpu3:239:239 [3] NCCL INFO Comm config Blocking set to 1 +gpu3:240:240 [4] NCCL INFO Comm config Blocking set to 1 +gpu3:242:242 [6] NCCL INFO Initialized NET plugin IB +gpu3:242:242 [6] NCCL INFO Assigned NET plugin IB to comm +gpu3:242:242 [6] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:242:242 [6] NCCL INFO Using network IB +gpu3:238:238 [2] NCCL INFO Initialized NET plugin IB +gpu3:238:238 [2] NCCL INFO Assigned NET plugin IB to comm +gpu3:238:238 [2] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:238:238 [2] NCCL INFO Using network IB +gpu3:242:242 [6] NCCL INFO DMA-BUF is available on GPU device 6 +gpu3:236:236 [0] NCCL INFO Initialized NET plugin IB +gpu3:238:238 [2] NCCL INFO DMA-BUF is available on GPU device 2 +gpu3:236:236 [0] NCCL INFO Assigned NET plugin IB to comm +gpu3:236:236 [0] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:236:236 [0] NCCL INFO Using network IB +gpu3:236:236 [0] NCCL INFO DMA-BUF is available on GPU device 0 +gpu3:238:238 [2] NCCL INFO ncclCommInitRankConfig comm 0x4872e440 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:242:242 [6] NCCL INFO ncclCommInitRankConfig comm 0x64e28690 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:236:236 [0] NCCL INFO ncclCommInitRankConfig comm 0x4d6d13a0 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:239:239 [3] NCCL INFO Initialized NET plugin IB +gpu3:239:239 [3] NCCL INFO Assigned NET plugin IB to comm +gpu3:239:239 [3] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:239:239 [3] NCCL INFO Using network IB +gpu3:243:243 [7] NCCL INFO Initialized NET plugin IB +gpu3:243:243 [7] NCCL INFO Assigned NET plugin IB to comm +gpu3:243:243 [7] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:243:243 [7] NCCL INFO Using network IB +gpu3:239:239 [3] NCCL INFO DMA-BUF is available on GPU device 3 +gpu3:241:241 [5] NCCL INFO Initialized NET plugin IB +gpu3:241:241 [5] NCCL INFO Assigned NET plugin IB to comm +gpu3:243:243 [7] NCCL INFO DMA-BUF is available on GPU device 7 +gpu3:241:241 [5] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:241:241 [5] NCCL INFO Using network IB +gpu3:239:239 [3] NCCL INFO ncclCommInitRankConfig comm 0x3c04cf10 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:241:241 [5] NCCL INFO DMA-BUF is available on GPU device 5 +gpu3:237:237 [1] NCCL INFO Initialized NET plugin IB +gpu3:237:237 [1] NCCL INFO Assigned NET plugin IB to comm +gpu3:237:237 [1] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:237:237 [1] NCCL INFO Using network IB +gpu3:243:243 [7] NCCL INFO ncclCommInitRankConfig comm 0x5904a050 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:237:237 [1] NCCL INFO DMA-BUF is available on GPU device 1 +gpu3:241:241 [5] NCCL INFO ncclCommInitRankConfig comm 0x3841f6f0 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:237:237 [1] NCCL INFO ncclCommInitRankConfig comm 0x4e6dc9e0 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:240:240 [4] NCCL INFO Initialized NET plugin IB +gpu3:240:240 [4] NCCL INFO Assigned NET plugin IB to comm +gpu3:240:240 [4] NCCL INFO Assigned GIN plugin GIN_IB_GDAKI to comm +gpu3:240:240 [4] NCCL INFO Using network IB +gpu3:240:240 [4] NCCL INFO DMA-BUF is available on GPU device 4 +gpu3:240:240 [4] NCCL INFO ncclCommInitRankConfig comm 0x52154ee0 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x6b0bbb24915a0c71 - Init START +gpu3:237:237 [1] NCCL INFO Bootstrap timings total 0.077728 (create 0.000058, send 0.000270, recv 0.000990, ring 0.076053, delay 0.000000) +gpu3:236:236 [0] NCCL INFO Bootstrap timings total 0.078318 (create 0.000049, send 0.000234, recv 0.001326, ring 0.015143, delay 0.000000) +gpu3:239:239 [3] NCCL INFO Bootstrap timings total 0.078040 (create 0.000047, send 0.000226, recv 0.001681, ring 0.075817, delay 0.000000) +gpu3:238:238 [2] NCCL INFO Bootstrap timings total 0.078405 (create 0.000050, send 0.000278, recv 0.000699, ring 0.076144, delay 0.000000) +gpu3:242:242 [6] NCCL INFO Bootstrap timings total 0.078472 (create 0.000048, send 0.000292, recv 0.000786, ring 0.076704, delay 0.000000) +gpu3:240:240 [4] NCCL INFO Bootstrap timings total 0.076708 (create 0.000046, send 0.000206, recv 0.000321, ring 0.075827, delay 0.000000) +gpu3:243:243 [7] NCCL INFO Bootstrap timings total 0.078020 (create 0.000054, send 0.000238, recv 0.000561, ring 0.076861, delay 0.000000) +gpu3:241:241 [5] NCCL INFO Bootstrap timings total 0.077911 (create 0.000053, send 0.000265, recv 0.000581, ring 0.075849, delay 0.000000) +gpu3:240:240 [4] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 4 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:242:242 [6] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 6 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:239:239 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:236:236 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:238:238 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:237:237 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 0-31,64-95. (GPU affinity = 0-31,64-95 ; CPU affinity = 0-127). +gpu3:243:243 [7] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 7 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:241:241 [5] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 5 is 32-63,96-127. (GPU affinity = 32-63,96-127 ; CPU affinity = 0-127). +gpu3:239:239 [3] NCCL INFO comm 0x3c04cf10 rank 11 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:238:238 [2] NCCL INFO comm 0x4872e440 rank 10 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:236:236 [0] NCCL INFO comm 0x4d6d13a0 rank 8 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:237:237 [1] NCCL INFO comm 0x4e6dc9e0 rank 9 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:239:239 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] 9/7/15->11->3 [2] -1/-1/-1->11->10 [3] 9/-1/-1->11->7 +gpu3:239:239 [3] NCCL INFO P2P Chunksize set to 131072 +gpu3:239:239 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:236:236 [0] NCCL INFO Trees [0] 9/4/12->8->0 [1] -1/-1/-1->8->10 [2] 9/-1/-1->8->4 [3] -1/-1/-1->8->10 +gpu3:236:236 [0] NCCL INFO P2P Chunksize set to 131072 +gpu3:238:238 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 8/-1/-1->10->9 [2] 11/-1/-1->10->9 [3] 8/-1/-1->10->9 +gpu3:236:236 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:238:238 [2] NCCL INFO P2P Chunksize set to 131072 +gpu3:241:241 [5] NCCL INFO comm 0x3841f6f0 rank 13 nRanks 16 nNodes 4 localRanks 4 localRank 1 MNNVL 0 +gpu3:238:238 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:240:240 [4] NCCL INFO comm 0x52154ee0 rank 12 nRanks 16 nNodes 4 localRanks 4 localRank 0 MNNVL 0 +gpu3:237:237 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/-1/-1->9->11 [2] 10/-1/-1->9->8 [3] 10/-1/-1->9->11 +gpu3:237:237 [1] NCCL INFO P2P Chunksize set to 131072 +gpu3:237:237 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:240:240 [4] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] -1/-1/-1->12->14 [2] 13/4/-1->12->-1 [3] -1/-1/-1->12->14 +gpu3:240:240 [4] NCCL INFO P2P Chunksize set to 131072 +gpu3:240:240 [4] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:242:242 [6] NCCL INFO comm 0x64e28690 rank 14 nRanks 16 nNodes 4 localRanks 4 localRank 2 MNNVL 0 +gpu3:243:243 [7] NCCL INFO comm 0x5904a050 rank 15 nRanks 16 nNodes 4 localRanks 4 localRank 3 MNNVL 0 +gpu3:241:241 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->15 [2] 14/-1/-1->13->12 [3] 14/-1/-1->13->15 +gpu3:242:242 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 12/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 12/-1/-1->14->13 +gpu3:241:241 [5] NCCL INFO P2P Chunksize set to 131072 +gpu3:242:242 [6] NCCL INFO P2P Chunksize set to 131072 +gpu3:243:243 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 13/-1/-1->15->11 [2] -1/-1/-1->15->14 [3] 13/7/-1->15->-1 +gpu3:243:243 [7] NCCL INFO P2P Chunksize set to 131072 +gpu3:242:242 [6] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:243:243 [7] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:241:241 [5] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 +gpu3:239:2229 [0] NCCL INFO [Proxy Service] Device 3 CPU core 20 +gpu3:236:2228 [0] NCCL INFO [Proxy Service] Device 0 CPU core 30 +gpu3:239:2233 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 21 +gpu3:236:2232 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 95 +gpu3:243:2239 [0] NCCL INFO [Proxy Service] Device 7 CPU core 103 +gpu3:240:2234 [0] NCCL INFO [Proxy Service] Device 4 CPU core 61 +gpu3:237:2235 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 80 +gpu3:238:2230 [0] NCCL INFO [Proxy Service] Device 2 CPU core 65 +gpu3:242:2238 [0] NCCL INFO [Proxy Service] Device 6 CPU core 34 +gpu3:240:2237 [0] NCCL INFO [Proxy Service UDS] Device 4 CPU core 63 +gpu3:242:2243 [0] NCCL INFO [Proxy Service UDS] Device 6 CPU core 42 +gpu3:243:2241 [0] NCCL INFO [Proxy Service UDS] Device 7 CPU core 40 +gpu3:238:2236 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 2 +gpu3:237:2231 [0] NCCL INFO [Proxy Service] Device 1 CPU core 79 +gpu3:241:2240 [0] NCCL INFO [Proxy Service] Device 5 CPU core 38 +gpu3:241:2242 [0] NCCL INFO [Proxy Service UDS] Device 5 CPU core 103 +gpu3:238:238 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 01/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 02/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 01/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 03/0 : 10[2] -> 11[3] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 01/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 03/0 : 14[6] -> 15[7] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 03/0 : 13[5] -> 14[6] via P2P/IPC +gpu3:243:2244 [0] NCCL INFO [Proxy Progress] Device 7 CPU core 107 +gpu3:243:243 [7] NCCL INFO Channel 00/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 02/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 0[0] [send] via NET/IB/1/GDRDMA +gpu3:240:2245 [0] NCCL INFO [Proxy Progress] Device 4 CPU core 33 +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:236:2246 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 67 +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 01/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 03/0 : 7[7] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 01/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 01/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 03/0 : 12[4] -> 13[5] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/IPC +gpu3:239:2247 [0] NCCL INFO [Proxy Progress] Device 3 CPU core 86 +gpu3:239:239 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 02/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 11[3] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:242:242 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:241:241 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:240:240 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:239:239 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:240:240 [4] NCCL INFO Channel 01/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 01/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:237:237 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:237:237 [1] NCCL INFO Channel 01/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +gpu3:236:236 [0] NCCL INFO Channel 01/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 03/0 : 13[5] -> 15[7] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 03/0 : 8[0] -> 10[2] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 03/0 : 12[4] -> 14[6] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 03/0 : 9[1] -> 11[3] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 01/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 4[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:238:238 [2] NCCL INFO Channel 03/0 : 10[2] -> 8[0] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 7[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 15[7] [send] via NET/IB/0/GDRDMA +gpu3:242:242 [6] NCCL INFO Channel 01/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 8[0] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 7[7] -> 15[7] [receive] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 7[7] [send] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [receive] via NET/IB/1/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [send] via NET/IB/1/GDRDMA +gpu3:242:242 [6] NCCL INFO Channel 03/0 : 14[6] -> 12[4] via P2P/IPC +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 3[3] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 3[3] [send] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 00/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:240:240 [4] NCCL INFO Channel 00/0 : 12[4] -> 8[0] [send] via NET/IB/1/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [receive] via NET/IB/0/GDRDMA +gpu3:236:236 [0] NCCL INFO Channel 02/0 : 8[0] -> 4[4] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 11[3] -> 7[7] [send] via NET/IB/0/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 15[7] -> 11[3] [send] via NET/IB/1/GDRDMA +gpu3:243:243 [7] NCCL INFO Channel 01/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Channel 03/0 : 15[7] -> 13[5] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 01/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Channel 00/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 03/0 : 11[3] -> 9[1] via P2P/IPC +gpu3:243:243 [7] NCCL INFO Channel 02/0 : 15[7] -> 14[6] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 00/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:239:239 [3] NCCL INFO Channel 02/0 : 11[3] -> 10[2] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 02/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 00/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:241:241 [5] NCCL INFO Channel 02/0 : 13[5] -> 12[4] via P2P/IPC +gpu3:242:242 [6] NCCL INFO Channel 03/0 : 14[6] -> 13[5] via P2P/IPC +gpu3:238:238 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:237:237 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/IPC +gpu3:240:240 [4] NCCL INFO Connected all trees +gpu3:241:241 [5] NCCL INFO Connected all trees +gpu3:236:236 [0] NCCL INFO Connected all trees +gpu3:237:237 [1] NCCL INFO Connected all trees +gpu3:243:243 [7] NCCL INFO Connected all trees +gpu3:242:242 [6] NCCL INFO Connected all trees +gpu3:239:239 [3] NCCL INFO Connected all trees +gpu3:238:238 [2] NCCL INFO Connected all trees +gpu3:243:243 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:243:243 [7] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:236:236 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:236:236 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:240:240 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:240:240 [4] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:239:239 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:239:239 [3] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:237:2248 [0] NCCL INFO [Proxy Progress] Device 1 CPU core 23 +gpu3:237:237 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:237:237 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:242:2249 [0] NCCL INFO [Proxy Progress] Device 6 CPU core 35 +gpu3:241:2250 [0] NCCL INFO [Proxy Progress] Device 5 CPU core 45 +gpu3:242:242 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:242:242 [6] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:241:241 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:241:241 [5] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:238:2251 [0] NCCL INFO [Proxy Progress] Device 2 CPU core 68 +gpu3:238:238 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512 +gpu3:238:238 [2] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 1 p2p channels per peer +gpu3:242:242 [6] NCCL INFO ncclCommInitRankConfig comm 0x64e28690 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId ae000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:240:240 [4] NCCL INFO ncclCommInitRankConfig comm 0x52154ee0 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId aa000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:242:242 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 14 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.04, topo 0.03, graphs 0.00, connections 0.21, rest 0.00) +gpu3:240:240 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 12 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.04, topo 0.03, graphs 0.00, connections 0.16, rest 0.05) +gpu3:243:243 [7] NCCL INFO ncclCommInitRankConfig comm 0x5904a050 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId af000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:243:243 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 15 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.03, topo 0.03, graphs 0.00, connections 0.16, rest 0.05) +gpu3:241:241 [5] NCCL INFO ncclCommInitRankConfig comm 0x3841f6f0 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId ab000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:241:241 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 13 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.03, topo 0.03, graphs 0.00, connections 0.21, rest 0.00) +gpu3:238:238 [2] NCCL INFO ncclCommInitRankConfig comm 0x4872e440 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 1b000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:236:236 [0] NCCL INFO ncclCommInitRankConfig comm 0x4d6d13a0 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 19000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:237:237 [1] NCCL INFO ncclCommInitRankConfig comm 0x4e6dc9e0 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 1a000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:238:238 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 10 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.04, topo 0.03, graphs 0.00, connections 0.21, rest 0.00) +gpu3:236:236 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 8 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.04, topo 0.03, graphs 0.00, connections 0.16, rest 0.05) +gpu3:239:239 [3] NCCL INFO ncclCommInitRankConfig comm 0x3c04cf10 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 1d000 commId 0x6b0bbb24915a0c71 - Init COMPLETE +gpu3:237:237 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 9 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.03, topo 0.03, graphs 0.00, connections 0.18, rest 0.03) +gpu3:239:239 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 11 nranks 16 total 0.35 (kernels 0.00, alloc 0.00, bootstrap 0.08, allgathers 0.04, topo 0.03, graphs 0.00, connections 0.16, rest 0.05) +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-08-01 05:04:01 TP12 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP12 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP12 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.80 GB +[2026-08-01 05:04:01 TP8 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP8 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP14 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP14 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP10 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP10 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP9 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP9 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP8 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.72 GB +[2026-08-01 05:04:01 TP10 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:04:01 TP9 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +[2026-08-01 05:04:01 TP14 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.86 GB +[2026-08-01 05:04:01 TP11 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP11 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP15 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP15 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP13 EP1] FlashInfer autotune completed. +[2026-08-01 05:04:01 TP13 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). +[2026-08-01 05:04:01 TP11 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.69 GB +[2026-08-01 05:04:01 TP15 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.77 GB +[2026-08-01 05:04:01 TP13 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=6.83 GB +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance. + warnings.warn( +[2026-08-01 05:05:31 TP12 EP1] Capture target decode CUDA graph end. elapsed=90.04 s, mem usage=1.37 GB, avail mem=5.43 GB. +[2026-08-01 05:05:31 TP12 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP12 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP8 EP1] Capture target decode CUDA graph end. elapsed=90.04 s, mem usage=1.37 GB, avail mem=5.35 GB. +[2026-08-01 05:05:31 TP8 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP8 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP11 EP1] Capture target decode CUDA graph end. elapsed=90.03 s, mem usage=1.37 GB, avail mem=5.32 GB. +[2026-08-01 05:05:31 TP11 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP11 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP14 EP1] Capture target decode CUDA graph end. elapsed=90.06 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:05:31 TP14 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP14 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP13 EP1] Capture target decode CUDA graph end. elapsed=90.05 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:05:31 TP13 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP13 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP10 EP1] Capture target decode CUDA graph end. elapsed=90.07 s, mem usage=1.37 GB, avail mem=5.50 GB. +[2026-08-01 05:05:31 TP10 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP10 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP9 EP1] Capture target decode CUDA graph end. elapsed=90.08 s, mem usage=1.37 GB, avail mem=5.46 GB. +[2026-08-01 05:05:31 TP9 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP9 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31 TP15 EP1] Capture target decode CUDA graph end. elapsed=90.07 s, mem usage=1.37 GB, avail mem=5.40 GB. +[2026-08-01 05:05:31 TP15 EP1] Init Unified RadixTree with components (, ) +[2026-08-01 05:05:31 TP15 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-08-01 05:05:31] Dummy health check server started in background thread at 0.0.0.0:30002 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_server_cmd.txt b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_server_cmd.txt new file mode 100644 index 0000000..636e480 --- /dev/null +++ b/docs/dsv4pro_pro6000d_2node_sglang/results/dsv4pro-phase2_5-20260801-130007/scout/service/worker_server_cmd.txt @@ -0,0 +1 @@ +docker run -d --name dsv4pro_pro6000d_2node_sglang_tp16_quick_map_worker --gpus all --network host --ipc host --shm-size 20g --ulimit memlock=-1 --ulimit stack=67108864 -v /data/hf_models/DeepSeek-V4-Pro:/data/hf_models/DeepSeek-V4-Pro:ro -v /data/hzy/sglang_cache/dsv4_pro_tp16:/root/.cache -e CUDA_VISIBLE_DEVICES=0\,1\,2\,3\,4\,5\,6\,7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -e NCCL_SOCKET_IFNAME=eth0 -e NCCL_IB_HCA==mlx5_0:1\,mlx5_3:1 -e NCCL_CROSS_NIC=1 -e NCCL_DEBUG=INFO -e SGLANG_SHARED_EXPERT_TP1=1 --device /dev/infiniband/rdma_cm --device /dev/infiniband/uverbs0 --device /dev/infiniband/uverbs3 --entrypoint python3 lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45 -m sglang.launch_server --model-path /data/hf_models/DeepSeek-V4-Pro --tp-size 16 --ep-size 2 --nnodes 2 --node-rank 1 --dist-init-addr 10.101.0.11:20002 --trust-remote-code --host 0.0.0.0 --port 30002 --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256 diff --git a/docs/dsv4pro_pro6000d_2node_sglang/推理优化计划.html b/docs/dsv4pro_pro6000d_2node_sglang/推理优化计划.html index e4156ef..ede1bc4 100644 --- a/docs/dsv4pro_pro6000d_2node_sglang/推理优化计划.html +++ b/docs/dsv4pro_pro6000d_2node_sglang/推理优化计划.html @@ -450,8 +450,11 @@ DeepSeek-V4-Pro / 双机 Pro6000D / SGLang RDMA 需求建模与并发拐点 -Phase 2.5 代码已实现；等待正式 Run，以 64K Scout 拟合 bytes/token、带宽平台和 400G 所需 Token TPS -阶段尚未完成，按门禁暂不创建实验/代码 HTML +Phase 2.5 已完成；Scout 5/5、Confirm 6/6，C=16 进入平台，拟合上限 80.32 Gbit/s/rail + +打开 Phase 2.5 实验档案
+打开 Phase 2.5 代码详解 +

@@ -799,11 +802,14 @@ Phase 2 已证明当前代表负载最高约 83.5 Gbit/s/rail，但单个固定 再自动选择平台前、拐点和最大稳定并发，对 64K → 1K 重复两次确认。

-本阶段必须同时给出每 Rail Mean/P95/Max、双 Rail 单向合计、Input/Output TPS、 -GPU 利用率、Rail 不均衡和 RDMA 错误增量。最终结论不是只报告“最大 C”，而是判断 -模型计算吞吐和 RDMA 链路哪个先进入平台，并形成一套换模型、TP/EP 或输入输出形状后 -可重复使用的带宽需求估算方法。正式结果完成前不创建 -phase2_5_exp.html 或 phase2_5_code.html。 +正式 Run dsv4pro-phase2_5-20260801-130007 已完成。64K→1 Scout 在 C=16 +达到约 2,984 input tok/s 与 79.90 Gbit/s/rail；C=32/64 均不再显著增长，拟合 +单 Rail 渐近上限为 80.32 Gbit/s。通信强度约为 3.332 MB/input-token/rail， +要达到单 Rail 400G 需约 15,006 input tok/s，约为当前平台的 5 倍。64K→1K +两轮确认同样在 C=16 后进入平台，说明当前是模型计算/实现吞吐先饱和，而非 RDMA 链路先饱和。 +完整方法、逐点结果和证据路径见 +Phase 2.5 实验档案 与 +Phase 2.5 代码详解。

7. Phase 3：时间线 Profiling（Nsight Systems 为主）