From 0e33209ae53f0f737cf945cd8675fed4cbf11f13 Mon Sep 17 00:00:00 2001 From: Zhiyi Hong <2497491955@qq.com> Date: Mon, 31 Aug 2026 17:53:30 +0800 Subject: [PATCH] fix: derive hybrid PD transfer bounds from full-attention layers --- README.md | 2 + docs/Kimi-K3_PP与DFlash迁移审计.md | 2 +- .../README.md | 14 +- .../config.env | 2 +- .../pp_dflash_files.json | 4 +- .../pp_dflash_integration.patch | 98 ++- .../image_build_kvbounds1_20260831.log | 76 +++ .../results/kvbounds1_delta_metadata.json | 8 + .../SOURCE_MANIFEST.json | 10 + .../kimi-dflash-kvbounds.Dockerfile | 5 + .../commands/p_0.cmd.txt | 1 + .../commands/p_1.cmd.txt | 1 + .../commands/p_2.cmd.txt | 1 + .../commands/p_3.cmd.txt | 1 + .../results/pd-dflash-20260831-1617/exit_code | 1 + .../pd-dflash-20260831-1617/logs/d_0.log | 1 + .../pd-dflash-20260831-1617/logs/d_1.log | 1 + .../pd-dflash-20260831-1617/logs/d_2.log | 1 + .../pd-dflash-20260831-1617/logs/d_3.log | 1 + .../pd-dflash-20260831-1617/logs/p_0.log | 606 +++++++++++++++++ .../pd-dflash-20260831-1617/logs/p_1.log | 628 +++++++++++++++++ .../pd-dflash-20260831-1617/logs/p_2.log | 553 +++++++++++++++ .../pd-dflash-20260831-1617/logs/p_3.log | 550 +++++++++++++++ .../pd-dflash-20260831-1617/logs/router.log | 1 + .../metadata/preflight.txt | 8 + .../commands/p_0.cmd.txt | 1 + .../commands/p_1.cmd.txt | 1 + .../commands/p_2.cmd.txt | 1 + .../commands/p_3.cmd.txt | 1 + .../pd-dflash-pwarm1-20260831-1652/exit_code | 1 + .../logs/d_0.log | 1 + .../logs/d_1.log | 1 + .../logs/d_2.log | 1 + .../logs/d_3.log | 1 + .../logs/p_0.log | 582 ++++++++++++++++ .../logs/p_1.log | 570 ++++++++++++++++ .../logs/p_2.log | 638 ++++++++++++++++++ .../logs/p_3.log | 594 ++++++++++++++++ .../logs/router.log | 1 + .../metadata/preflight.txt | 8 + 40 files changed, 4967 insertions(+), 11 deletions(-) create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/image_build_kvbounds1_20260831.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds1_delta_metadata.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/SOURCE_MANIFEST.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/kimi-dflash-kvbounds.Dockerfile create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/exit_code create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_2.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_2.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/router.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/metadata/preflight.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/exit_code create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_2.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_2.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/router.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/metadata/preflight.txt diff --git a/README.md b/README.md index 7e8e2fc..45d070f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # sskj — 多平台大模型推理性能基准测试项目 +**更新（2026-08-31 17:50:51 CST）**：修复 Kimi 混合注意力缓存池在 PD 初始化时缺少 `end_layer` 的兼容问题，按完整注意力层偏移和本地层数确定传输范围，保留普通池与 layer-shard 路径。新增回归后 24 项 CPU 测试通过，八节点修复镜像摘要一致；P 组已完成预热和 Mooncake 注册，健康检查返回 200，D 组正在验证启动。当前 Run 为 `pd-dflash-kvbounds1-20260831-1740`，尚无 GSM8K 验收结果。详见 `experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md`。 + **更新（2026-08-31 17:05:00 CST）**：新增 Kimi-K3 PP8 + DFlash PD 适配验证入口。基于 SGLang PR #33863 固定源码，接通 PP 分段 hidden 投影、P 侧 prompt draft KV 生成、D 侧输入生命周期与 TP4→TP32 的 draft GQA KV 传输，保留 Kimi SM120 FlashInfer MXFP4 接入。修复 P 普通预热误带 DFlash verify metadata 的启动问题，23 项 CPU 回归通过；修复镜像已同步 601–608，服务级验证正在进行，尚无 GSM8K 结果。配置为 P TP4/PP8/EP4、D TP32/PP1/EP4、BF16 KV、8K Chunk，计划固定 64 题 C1/C8。详见 `experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md`。 **更新（2026-08-27 13:53:26 CST）**：完成 Kimi-K3 八节点标准 PD 第一阶段。P 组 601-604 使用 PP8×TP4×EP4、FlashInfer MXFP4、Chunk 8K，D 组 605-608 使用 PP1×TP32×EP32、Marlin，通过 Mooncake 0.3.12.post1 和 4 Rail RDMA 传输；统一 P/D `page_size=64` 后，16K→1 与 16K→512 的 C1/C8 共 91/91 请求成功。代表结果：16K→1 C8 Input TPS 6364.31、TTFT P50/P95 20.555/21.345 秒；16K→512 C8 TPOT P50/P95 63.20/66.55 ms。详见 `experiments/pro6000/kimi3_pro6000_pd_pp8_standard/README.md`。 diff --git a/docs/Kimi-K3_PP与DFlash迁移审计.md b/docs/Kimi-K3_PP与DFlash迁移审计.md index 34cc422..0d1fd5e 100644 --- a/docs/Kimi-K3_PP与DFlash迁移审计.md +++ b/docs/Kimi-K3_PP与DFlash迁移审计.md @@ -6,7 +6,7 @@ **优先推进 DFlash 的 PP + PD 适配。** #33863 的分段投影设计能复用于 DFlash，但需要补齐 DFlash worker、调度入口和异构 TP 下的 draft KV 传输。 -当前已实现 DFlash PP worker、Kimi 跨 stage capture、PD 输入衔接、异构 TP 的 draft KV 传输，以及 Kimi FlashInfer 布局/SiTU 接入。首轮 P 启动暴露普通预热误带 verify metadata 的问题，已修复并增加回归，23 项 CPU 测试通过。SM120 编译及 Kimi SiTU GPU 集成回归已通过。修复镜像已同步全部八节点，Run `pd-dflash-pwarm1-20260831-1652` 正在重新验证服务启动；尚无 PD 请求和 GSM8K 结果。详见 [实现进度与命令](evidence/kimi_k3_pd_dflash/README.md)。 +当前已实现 DFlash PP worker、Kimi 跨 stage capture、PD 输入衔接、异构 TP 的 draft KV 传输，以及 Kimi FlashInfer 布局/SiTU 接入。P 端两次启动分别暴露普通预热误带 verify metadata、混合池缺少 `end_layer` 的问题，均已针对实际调用点修复，24 项 CPU 测试通过。SM120 编译及 Kimi SiTU GPU 集成回归已通过。Run `pd-dflash-kvbounds1-20260831-1740` 的 P 组健康检查已返回 200，D 组正在启动；尚无跨组请求和 GSM8K 结果。详见 [实现进度与命令](evidence/kimi_k3_pd_dflash/README.md)。 暂停 EAGLE3 baseline。KV cache 保持 BF16，chunk 保持 8192。FlashInfer 使用官方 0.6.18 加已合并 #4460 的显式 backport；原版 0.6.18 wheel 尚未包含所需 CUTLASS SiTU 接口。 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md index c33d265..eedc8e4 100644 --- a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md @@ -1,6 +1,6 @@ # Kimi-K3 PP + DFlash 适配进度 -2026-08-31：PP worker、PD 输入衔接、异构 TP 传输和 Kimi FlashInfer 接入已完成首版；SM120 编译及 Kimi SiTU GPU 集成回归通过。首轮八节点部署在 P 侧预热阶段失败，已修复普通预热误带 DFlash verify metadata 的问题；本地、601 构建镜像及 602 增量导入容器的 23 项 CPU 回归通过。正在重新验证服务启动，尚无 PD 请求或 GSM8K 结果。 +2026-08-31：PP worker、PD 输入衔接、异构 TP 传输和 Kimi FlashInfer 接入已完成首版；SM120 编译及 Kimi SiTU GPU 集成回归通过。两项启动兼容修复及 24 项 CPU 回归已完成。Run `pd-dflash-kvbounds1-20260831-1740` 的 P 组已通过预热、Mooncake 初始化和健康检查（HTTP 200），D 组正在启动；尚无跨组请求或 GSM8K 结果。 ## 源码和环境 @@ -46,16 +46,16 @@ python3 -m unittest discover -s test/registered/disaggregation -v | 回归 | 数量 | 主要覆盖 | |---|---:|---| | 传输 | 7 | 144 组 KV-head/TP 组合、实际 P-TP4→D-TP32 全 32 rank、逐字节复制、越界/重复写拒绝、注册往返 | -| PP 与 PD 输入 | 12 | PP1/2/4/8/16 capture 归属、边界 residual、空 capture stage、BF16/FP32 分段投影、单次 norm、末 stage 写 KV、输入生命周期、P 普通预热不创建 verify metadata | +| PP 与 PD 输入 | 13 | PP1/2/4/8/16 capture 归属、边界 residual、空 capture stage、BF16/FP32 分段投影、单次 norm、末 stage 写 KV、输入生命周期、P 普通预热不创建 verify metadata、混合池/普通池传输层范围 | | Kimi MoE 合并 | 4 | 两类 gate/up 布局、SiTU 参数、activation 白名单、非连续输入、保留 SwigluStep、API 能力检查 | -23 项在本地、601 修复镜像和 602 导入容器中通过，并完成真实 DFlashWorkerV2、MooncakeKVManager 导入。8 条启动命令的 CLI 解析及两类服务的参数后处理已检查。CPU 测试对实际方法作 AST 提取，使用 CPU tensor 或记录型 engine，核验数学与调用契约，不替代服务级验证。 +24 项在本地与 601 `kvbounds1` 镜像中通过。此前 `pwarm1` 镜像的 23 项也在 602 导入容器中通过，并完成真实 DFlashWorkerV2、MooncakeKVManager 导入。8 条启动命令的 CLI 解析及两类服务的参数后处理已检查。CPU 测试对实际方法作 AST 提取，使用 CPU tensor 或记录型 engine，核验数学与调用契约，不替代服务级验证。 601 GPU6 的 `test_kimi_k3_sm120_situ_layout_and_noncontiguous_input` 已通过：检查 Kimi gate/up 及 scale 布局、SiTU(4,25)、非连续输入，以及 SGLang adapter 与直接 FlashInfer 调用的输出一致性。这是小 shape 的集成回归；端到端正确性由后续 PD/GSM8K 检验。 修复前镜像 `local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460` 的独立 GPU 复测通过，Mooncake CUDA engine 导入成功，GPU 测试 1 passed、25.28 秒（复用 JIT 缓存）。 -当前镜像为 `local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1`，只追加 P 预热修复，不改变 GPU kernel。Linux/amd64 manifest 为 `sha256:281ccb2666a38acd539e6fbb5d55d3682eb2af9ac9089f67e9bf92a8ddd822eb`，image config 为 `sha256:8ab5eee9902556bcba2a2b439a9fa1b14d93e0554503754fb7ea74dad6c3cf79`；OCI index 为 `sha256:d30d68757014b973c2edd5d061cc3fd55660a4c68cc3d0af9050fcb7e64c4a78`。区分这三个摘要，不将 Docker 不同模式显示的 ID 当成代码不一致。 +当前镜像为 `local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1`，在 `pwarm1` 上只替换 Python 层范围处理及回归测试，不改变 FlashInfer 层或 GPU kernel。Linux/amd64 manifest 为 `sha256:e74c9b1e7fb6a9509db32576d2c9196b42ef329118909eeebd620cb9db2f91e9`，image config 为 `sha256:26584138eb1e097183ce167cbffe4198be0611e0cfde7422cbe692100d865982`；OCI index 为 `sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f`。区分这三个摘要，不将 Docker 不同模式显示的 ID 当成代码不一致。 601 证据均位于实验目录 `results/`： @@ -73,6 +73,8 @@ python3 -m unittest discover -s test/registered/disaggregation -v - `pd-dflash-20260831-1617/logs/p_0.log` 至 `p_3.log`：首轮 P 预热报错原始证据；该 Run 未进入请求测试，退出码 1。 - `image_build_pwarm1_20260831.log`：预热修复镜像构建及 23 项 CPU 回归。 - `pwarm1_602_cpu.log`、`pwarm1_602_inspect.json`：约 49 MiB 增量包导入后的容器回归与平台摘要。 +- `pd-dflash-pwarm1-20260831-1652/logs/p_0.log`：P 预热完成后，在初始化传输管理器时发现 `HybridLinearKVPool.end_layer` 缺失；未进入 D 启动和请求测试。 +- `image_build_kvbounds1_20260831.log`、`kvbounds_image_context/`：复用原 FlashInfer 层、只修改 Python 的构建配方及 24 项回归。 已通过 compileall、Black 和 Ruff 的未定义变量/语法检查。 @@ -86,7 +88,7 @@ cd /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation DRY_RUN=1 RUN_ID=command-audit-20260831 bash deploy_pd_dflash.sh all ``` -首轮已执行 `all`，在 P 侧启动阶段退出。修复镜像同步完成后使用新 Run ID 重试。正式顺序为 `start` → `smoke` → `bench` → `logs` → `stop`，也可用 `all` 串行执行。所有操作使用同一个 `RUN_ID`。失败时保留本任务容器和日志，入口不会自动杀其他实验。 +此前两轮均在 P 侧启动阶段退出，日志已保存、失败容器已清理。修复镜像同步完成后使用新 Run ID 重试。正式顺序为 `start` → `smoke` → `bench` → `logs` → `stop`，也可用 `all` 串行执行。所有操作使用同一个 `RUN_ID`。失败时保留本任务容器和日志，入口不会自动杀其他实验。 配置选择：P 为 TP4/PP8/EP4，D 为 TP32/PP1/EP4，两侧 FlashInfer；KV 为 BF16，chunk=8192，page=64。为 C1/C8 评测将活跃请求与 Decode Graph 上限设为 8；未额外改变模型 context 上限。 @@ -102,4 +104,6 @@ GSM8K 为原数据集前 64 题，沿用历史客户端的 5-shot、temperature= `base_runner.py::_dummy_run` 已将 PD Prefill target 设为普通 Decode 预热，但随后仍创建 `DFlashVerifyInput`。普通 Triton Attention 读取 `kv_indptr` 时因此报 `AttributeError`。修复让该分支的 `spec_info=None`，D 侧真正的 TARGET_VERIFY 路径不变。没有强行添加字段，也没有修改 DFlash 接受算法。 +第二轮通过预热后，`PrefillBootstrapQueue._init_kv_manager` 直接访问混合池不存在的 `end_layer`。修复以现有的完整注意力层偏移加 `full_layer_nums` 得到传输范围，KDA 状态和 draft KV 仍分别使用明确的全局层 ID。新增回归覆盖 MLA/MHA 混合池、空完整注意力 stage、普通池和 layer-shard 分支，不将 K/V 两组条目误计为两倍层数。 + 镜像同步须检查 Docker 所在根盘，而不只检查模型盘 `/data`。本镜像层展开约 34.4GB，压缩内容约 15GB；共享层会减少增量占用。607 已按用户授权删除两个无容器引用的 vLLM 镜像，根盘恢复到约 107GB，原镜像元数据保存在 `results/607_vllm_images_before_delete_20260831.json`。用户清理 606 后其根盘恢复到约 91GB。601 使用 `ctr images export` 将既有 OCI 压缩层直接导出到 `/data`，其旧 vLLM 镜像尚未删除。 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/config.env b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/config.env index 9e4183b..917c5b6 100644 --- a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/config.env +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/config.env @@ -1,5 +1,5 @@ EXPERIMENT=kimi3_pro6000_pd_dflash_validation -PD_IMAGE=local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 +PD_IMAGE=local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 MODEL_PATH=/data/hf_models/Kimi-K3 DRAFT_MODEL_PATH=/data/hf_models/Kimi-K3-DFlash P_NODES=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4) diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_files.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_files.json index 8a1546e..9e1dc5e 100644 --- a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_files.json +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_files.json @@ -27,7 +27,7 @@ }, { "path": "python/sglang/srt/disaggregation/prefill.py", - "sha256": "d0175b3df8eddd58fbfa41805d0928d614e412cf74055e57aade82578b174bfd" + "sha256": "d2e70dd6d8606e218da15a987a5e3a2e82f5ef77ab43f63f1feaf90ad5b8d286" }, { "path": "python/sglang/srt/disaggregation/utils.py", @@ -67,7 +67,7 @@ }, { "path": "test/registered/disaggregation/test_dflash_pp_context.py", - "sha256": "3f3022afa4acd7f66e14caf6bbbfb88b1f2e9320d087300aa78c9d9e20d3a321" + "sha256": "955e45f288b060371a26f7f5a1e7490e3bacc1c8941ab7f2dbca2e3bc98b4bc9" }, { "path": "test/registered/disaggregation/test_flashinfer_kimi_merge.py", diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_integration.patch b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_integration.patch index 83cf672..89d62c9 100644 --- a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_integration.patch +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/pp_dflash_integration.patch @@ -424,7 +424,33 @@ diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/dis build_kv_layer_ids, build_staging_slot_metadata, get_dsv4_c128_state_indices, -@@ -252,6 +253,18 @@ +@@ -112,6 +113,13 @@ + ) + + ++def _transfer_end_layer(*, pool, start_layer: int) -> int: ++ if isinstance(pool, HybridLinearKVPool): ++ # Match the dense full-attention offset used by _transfer_start_layer. ++ return start_layer + pool.full_layer_nums ++ return pool.end_layer ++ ++ + def maybe_release_metadata_buffer( + req: Req, allocator: ReqToMetadataIdxAllocator + ) -> None: +@@ -233,7 +241,10 @@ + kv_args.prefill_end_layer = ( + kv_args.prefill_start_layer + len(kv_data_ptrs) + if layer_shard_enabled +- else self.token_to_kv_pool.end_layer ++ else _transfer_end_layer( ++ pool=self.token_to_kv_pool, ++ start_layer=kv_args.prefill_start_layer, ++ ) + ) + + draft_kv_pool = self.draft_token_to_kv_pool if transfer_draft_cache else None +@@ -252,6 +263,18 @@ kv_args.kv_data_ptrs = kv_data_ptrs kv_args.kv_data_lens = kv_data_lens kv_args.kv_item_lens = kv_item_lens @@ -1203,7 +1229,7 @@ diff --git a/test/registered/disaggregation/test_dflash_pp_context.py b/test/reg new file mode 100644 --- /dev/null +++ b/test/registered/disaggregation/test_dflash_pp_context.py -@@ -0,0 +1,431 @@ +@@ -0,0 +1,499 @@ +"""CPU contract tests of the production PP methods; no model/RDMA emulation claim.""" + +import ast @@ -1301,6 +1327,74 @@ new file mode 100644 + + +class TestPPContext(unittest.TestCase): ++ def test_prefill_transfer_bounds_for_hybrid_and_ordinary_pools(self): ++ class HybridPool: ++ def __init__(self, start, count): ++ self.start_layer = start ++ self.full_layer_nums = count ++ ++ path = "disaggregation/prefill.py" ++ globals_dict = {"HybridLinearKVPool": HybridPool} ++ start_fn = load_function(path, "_transfer_start_layer", globals_dict) ++ end_fn = load_function(path, "_transfer_end_layer", globals_dict) ++ tree = ast.parse((SRT / path).read_text()) ++ cls = next( ++ n ++ for n in tree.body ++ if isinstance(n, ast.ClassDef) and n.name == "PrefillBootstrapQueue" ++ ) ++ method = next( ++ n ++ for n in cls.body ++ if isinstance(n, ast.FunctionDef) and n.name == "_init_kv_manager" ++ ) ++ assignment = next( ++ n ++ for n in method.body ++ if isinstance(n, ast.Assign) ++ and any( ++ isinstance(t, ast.Attribute) and t.attr == "prefill_end_layer" ++ for t in n.targets ++ ) ++ ) ++ code = compile( ++ ast.fix_missing_locations(ast.Module(body=[assignment], type_ignores=[])), ++ str(SRT / path), ++ "exec", ++ ) ++ config = SimpleNamespace(full_attention_layer_ids=[2, 6, 10, 14, 18]) ++ cases = [ ++ (HybridPool(0, 2), False, 2, 0, 2), ++ (HybridPool(8, 2), False, 2, 2, 4), ++ ( ++ HybridPool(8, 2), ++ False, ++ 4, ++ 2, ++ 4, ++ ), # Separate K/V entries do not double layer count. ++ (HybridPool(20, 0), False, 0, 5, 5), ++ (SimpleNamespace(start_layer=8, end_layer=16), False, 3, 8, 16), ++ (SimpleNamespace(start_layer=8, end_layer=16), True, 3, 8, 11), ++ ] ++ for pool, sharded, entries, start, end in cases: ++ with self.subTest( ++ pool=type(pool).__name__, sharded=sharded, entries=entries ++ ): ++ self.assertEqual(start_fn(pool=pool, hf_text_config=config), start) ++ args = SimpleNamespace(prefill_start_layer=start) ++ exec( ++ code, ++ { ++ "kv_args": args, ++ "kv_data_ptrs": [0] * entries, ++ "layer_shard_enabled": sharded, ++ "self": SimpleNamespace(token_to_kv_pool=pool), ++ "_transfer_end_layer": end_fn, ++ }, ++ ) ++ self.assertEqual(args.prefill_end_layer, end) ++ + def test_prefill_dummy_forward_has_no_verify_metadata(self): + path = SRT / "model_executor/runner/base_runner.py" + tree = ast.parse(path.read_text()) diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/image_build_kvbounds1_20260831.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/image_build_kvbounds1_20260831.log new file mode 100644 index 0000000..a891372 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/image_build_kvbounds1_20260831.log @@ -0,0 +1,76 @@ +#0 building with "default" instance using docker driver + +#1 [internal] load build definition from kimi-dflash-kvbounds.Dockerfile +#1 transferring dockerfile: 458B done +#1 DONE 0.0s + +#2 [internal] load metadata for docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 +#2 DONE 1.1s + +#3 [internal] load .dockerignore +#3 transferring context: 2B done +#3 DONE 0.0s + +#4 [1/5] FROM docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1@sha256:d30d68757014b973c2edd5d061cc3fd55660a4c68cc3d0af9050fcb7e64c4a78 +#4 resolve docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1@sha256:d30d68757014b973c2edd5d061cc3fd55660a4c68cc3d0af9050fcb7e64c4a78 0.1s done +#4 ... + +#5 [internal] load build context +#5 transferring context: 82.52kB done +#5 DONE 0.1s + +#4 [1/5] FROM docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1@sha256:d30d68757014b973c2edd5d061cc3fd55660a4c68cc3d0af9050fcb7e64c4a78 +#4 DONE 5.3s + +#6 [2/5] COPY prefill.py /opt/kimi-dflash/python/sglang/srt/disaggregation/prefill.py +#6 DONE 0.0s + +#7 [3/5] COPY test_dflash_pp_context.py /opt/kimi-dflash/test/registered/disaggregation/test_dflash_pp_context.py +#7 DONE 0.1s + +#8 [4/5] COPY SOURCE_MANIFEST.json /opt/kimi-dflash/SOURCE_MANIFEST.json +#8 DONE 0.0s + +#9 [5/5] RUN python3 -m unittest discover -s /opt/kimi-dflash/test/registered/disaggregation -v +#9 5.047 test_capture_ownership_including_boundaries (test_dflash_pp_context.TestPPContext.test_capture_ownership_including_boundaries) ... ok +#9 5.050 test_invalid_capture_configuration (test_dflash_pp_context.TestPPContext.test_invalid_capture_configuration) ... ok +#9 5.050 test_kimi_boundary_uses_next_stage_weights (test_dflash_pp_context.TestPPContext.test_kimi_boundary_uses_next_stage_weights) ... ok +#9 8.108 test_last_stage_without_capture_uses_incoming_context (test_dflash_pp_context.TestPPContext.test_last_stage_without_capture_uses_incoming_context) ... ok +#9 8.116 test_missing_incoming_context_fails_before_kv_write (test_dflash_pp_context.TestPPContext.test_missing_incoming_context_fails_before_kv_write) ... ok +#9 8.117 test_nonfinal_pool_is_minimal_without_mutating_target_config (test_dflash_pp_context.TestPPContext.test_nonfinal_pool_is_minimal_without_mutating_target_config) ... ok +#9 8.144 test_only_prefill_skips_draft_graph_initialization (test_dflash_pp_context.TestPPContext.test_only_prefill_skips_draft_graph_initialization) ... ok +#9 8.165 test_pd_input_builder_and_spec_dispatch (test_dflash_pp_context.TestPPContext.test_pd_input_builder_and_spec_dispatch) ... ok +#9 8.171 test_pp8_boundary_capture (test_dflash_pp_context.TestPPContext.test_pp8_boundary_capture) ... ok +#9 8.173 test_pp8_projection_empty_stages_float32_and_bf16 (test_dflash_pp_context.TestPPContext.test_pp8_projection_empty_stages_float32_and_bf16) ... ok +#9 8.181 test_pp_proxy_is_consumed_once_even_on_forward_failure (test_dflash_pp_context.TestPPContext.test_pp_proxy_is_consumed_once_even_on_forward_failure) ... ok +#9 8.207 test_prefill_dummy_forward_has_no_verify_metadata (test_dflash_pp_context.TestPPContext.test_prefill_dummy_forward_has_no_verify_metadata) ... ok +#9 8.213 test_prefill_transfer_bounds_for_hybrid_and_ordinary_pools (test_dflash_pp_context.TestPPContext.test_prefill_transfer_bounds_for_hybrid_and_ordinary_pools) ... ok +#9 8.248 test_activation_selection_accepts_situ_and_legacy_enum (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_activation_selection_accepts_situ_and_legacy_enum) ... ok +#9 8.260 test_capability_requires_public_parameters (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_capability_requires_public_parameters) ... ok +#9 8.270 test_runner_preserves_swiglu_step_and_old_api (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_runner_preserves_swiglu_step_and_old_api) ... ok +#9 8.275 test_weight_layout_and_situ_parameters (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_weight_layout_and_situ_parameters) ... ok +#9 8.297 test_all_gqa_shards_and_replicas_byte_exact (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_all_gqa_shards_and_replicas_byte_exact) ... ok +#9 8.375 test_flat_stride_mismatch_rejected (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_flat_stride_mismatch_rejected) ... ok +#9 8.375 test_invalid_metadata_rejected_before_copy (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_invalid_metadata_rejected_before_copy) ... ok +#9 8.375 test_mooncake_mixed_entries_match_layers_before_transfer (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_mooncake_mixed_entries_match_layers_before_transfer) ... ok +#9 8.410 test_real_kimi_dflash_page64_tp4_to_tp32 (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_real_kimi_dflash_page64_tp4_to_tp32) ... ok +#9 8.411 test_replicated_mla_is_flat_and_coalesced (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_replicated_mla_is_flat_and_coalesced) ... ok +#9 8.411 test_wire_roundtrip (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_wire_roundtrip) ... ok +#9 8.411 +#9 8.411 ---------------------------------------------------------------------- +#9 8.411 Ran 24 tests in 3.365s +#9 8.411 +#9 8.411 OK +#9 DONE 9.2s + +#10 exporting to image +#10 exporting layers +#10 exporting layers 0.2s done +#10 exporting manifest sha256:e74c9b1e7fb6a9509db32576d2c9196b42ef329118909eeebd620cb9db2f91e9 0.0s done +#10 exporting config sha256:26584138eb1e097183ce167cbffe4198be0611e0cfde7422cbe692100d865982 done +#10 exporting attestation manifest sha256:6d9e087e2e30bfb53545cdfe1a315bceda337ac57ef4cd07f0b209546a7e3caf done +#10 exporting manifest list sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f done +#10 naming to docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 done +#10 unpacking to docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 +#10 unpacking to docker.io/local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 0.2s done +#10 DONE 0.7s diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds1_delta_metadata.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds1_delta_metadata.json new file mode 100644 index 0000000..69cf9ed --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds1_delta_metadata.json @@ -0,0 +1,8 @@ +{ + "archive": "sglang-kimi-pp-dflash-kvbounds1.delta.oci.tar", + "bytes": 133120, + "sha256": "800da9d85d186c21f41d1fe8681529ad3e658047786ae7e1bad92ec594939050", + "required_base_index": "d30d68757014b973c2edd5d061cc3fd55660a4c68cc3d0af9050fcb7e64c4a78", + "image_index": "7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f", + "linux_amd64_manifest": "e74c9b1e7fb6a9509db32576d2c9196b42ef329118909eeebd620cb9db2f91e9" +} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/SOURCE_MANIFEST.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/SOURCE_MANIFEST.json new file mode 100644 index 0000000..1a83ec2 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/SOURCE_MANIFEST.json @@ -0,0 +1,10 @@ +{ + "repository": "https://github.com/sgl-project/sglang", + "base_commit": "6465a6f3d3b6c8b7fee40fba0fdc09cf5e9ca1c5", + "base_pull_request": 33863, + "snapshot_scope": "python directory; all 4619 upstream blobs verified against git tree before edits", + "local_changes": "Mixed MLA/GQA transport; Kimi PP boundary capture; DFlash PP worker and PD input wiring; public FlashInfer SiTU adapter rebased from b95b534; PD-prefill warmup without verify metadata; dense hybrid prefill transfer bounds; 24 CPU regressions", + "status": "Experimental source snapshot; see external runtime logs for validation results", + "flashinfer_target": "official flashinfer-python 0.6.18 plus unchanged merged upstream SiTU commit b460bc00cb373541102d2155aec35bd626e522ce; old companion packages removed; standard JIT; version checks enabled", + "gpu_tested": false +} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/kimi-dflash-kvbounds.Dockerfile b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/kimi-dflash-kvbounds.Dockerfile new file mode 100644 index 0000000..04c2c7b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/kvbounds_image_context/kimi-dflash-kvbounds.Dockerfile @@ -0,0 +1,5 @@ +FROM local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 +COPY prefill.py /opt/kimi-dflash/python/sglang/srt/disaggregation/prefill.py +COPY test_dflash_pp_context.py /opt/kimi-dflash/test/registered/disaggregation/test_dflash_pp_context.py +COPY SOURCE_MANIFEST.json /opt/kimi-dflash/SOURCE_MANIFEST.json +RUN python3 -m unittest discover -s /opt/kimi-dflash/test/registered/disaggregation -v diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_0.cmd.txt new file mode 100644 index 0000000..dd28f5f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.1 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_1.cmd.txt new file mode 100644 index 0000000..690c412 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.2 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_2.cmd.txt new file mode 100644 index 0000000..b3c93eb --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.3 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_3.cmd.txt new file mode 100644 index 0000000..3aba7cb --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/commands/p_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.4 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/exit_code b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/exit_code new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/exit_code @@ -0,0 +1 @@ +1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_0.log new file mode 100644 index 0000000..8a2955b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_0.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_0 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_1.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_1.log new file mode 100644 index 0000000..f647e30 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_1.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_2.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_2.log new file mode 100644 index 0000000..c1ed8e8 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_2.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_2 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_3.log new file mode 100644 index 0000000..857ef64 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/d_3.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_3 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_0.log new file mode 100644 index 0000000..a09bfdb --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_0.log @@ -0,0 +1,606 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 08:19:04] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:19:04] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:19:04] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:19:08] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/root/.cache/sglang). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:19:08] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 0, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 1006133203, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:08] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:14] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:14] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:15] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:15] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:15] CommonKVBootstrapServer started successfully on 0.0.0.0:28800 +[2026-08-31 08:19:15] No HuggingFace chat template found +[2026-08-31 08:19:15] No chat template found, defaulting to 'string' content format +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:40 PP0 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:40 PP0 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:40 PP1 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:40 PP1 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:41] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:41] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +/opt/kimi-dflash/python/sglang/srt/utils/hf_transformers/tokenizer.py:433: UserWarning: Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead. + warnings.warn( +[2026-08-31 08:19:42] Applying special tokens cache patch for Kimi tokenizer: +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:43 PP0 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:43 PP0 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:43 PP1 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:43 PP1 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:44 PP1 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:44 PP1 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:44 PP0 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:44 PP0 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:44 PP0 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:44 PP0 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:44 PP0 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:44 PP0 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:44 PP1 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:44 PP1 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:44.610937 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:44.611007 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:44.611018 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:44.611047 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15416 +I0831 08:19:44.611119 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:44.635780 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:44.640407 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:44.645533 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:44.651093 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:44.653239 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:44.653410 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:44.653460 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.659595 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:44.660910 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:44.660934 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.686170 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:44.687633 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:44.687659 234 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:44.690487 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:44.690577 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:44.690590 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:44.690623 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16992 +I0831 08:19:44.690739 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:44.696539 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:44.702077 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:44.708596 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:44.714198 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:44.715652 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:44.715683 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.716519 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:44.720211 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:44.720307 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:44.720363 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.742235 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:44.743659 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:44.743794 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:44 PP1 TP3 EP3] Init torch distributed begin. +I0831 08:19:44.746265 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:44.747831 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:44.747864 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.774227 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:44.775617 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:44.775664 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.802265 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:44.803647 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:44.803682 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:44.830219 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:44.831576 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:44.831722 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:44 PP0 TP2 EP2] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:45 PP1 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:45 PP1 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:47 PP1 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:47 PP1 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:47 PP0 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:47 PP0 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:48.110203 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:48.110299 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:48.110312 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:48.110345 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16959 +I0831 08:19:48.110463 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:48.133812 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:48.141039 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:48.148154 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:48.174034 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:48.176815 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:48.176911 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:48.176966 232 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:48.184208 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:48.184293 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:48.184303 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:48.184334 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15689 +I0831 08:19:48.184446 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:48.189126 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:48.202255 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:48.207980 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:48.208104 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.212862 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:48.219226 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:48.226087 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:48.228790 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:48.228883 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:48.228940 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.234185 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:48.235910 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:48.235941 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.236129 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:48.237536 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:48.237563 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.262290 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:48.262290 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:48.263664 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:48.263707 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.263696 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:48.263720 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.271297 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:48.272724 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:48.272850 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:48 PP1 TP1 EP1] Init torch distributed begin. +I0831 08:19:48.290222 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:48.291606 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:48.291636 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:48.318179 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:48.319557 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:48.319695 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:48 PP0 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:19:48 PP1 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:48 PP1 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:48 PP0 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:48 PP0 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:48 PP0 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:48 PP0 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:49.029664 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:49.029788 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:49.029805 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:49.029848 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15241 +I0831 08:19:49.029989 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:49.036382 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:49.042028 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:49.048030 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:49.054361 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:49.057052 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:49.057152 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:49.057209 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.064553 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:49.065986 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:49.066015 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.090340 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:49.091794 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:49.091820 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.118383 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:49.119796 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:49.119820 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.146345 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:49.147733 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:49.147855 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:49 PP1 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:49.153810 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:49.153906 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:49.153923 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:49.153959 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15372 +I0831 08:19:49.154084 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:49.154865 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:49.154965 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:49.154982 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:49.155017 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16274 +I0831 08:19:49.155133 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:49.160967 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:49.166205 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:49.171423 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:49.177234 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:49.178143 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:49.179472 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:49.179550 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:49.179596 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.185333 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:49.186076 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:49.187390 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:49.187415 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.192319 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:49.194130 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:49.195423 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:49.195447 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.217983 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:49.220708 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:49.220796 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:49.220852 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.222326 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:49.223742 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:49.223771 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.228039 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:49.229475 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:49.229503 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.236876 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:49.238233 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:49.238261 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.250272 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:49.251724 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:49.251840 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:49 PP0 TP1 EP1] Init torch distributed begin. +I0831 08:19:49.262301 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:49.263700 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:49.263744 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.290270 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:49.291630 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:49.291772 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:49 PP0 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:49 PP1 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:49 PP1 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:49.805439 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:49.805500 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:49.805508 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:19:49.805528 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15061 +I0831 08:19:49.805600 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:49.809037 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:49.812366 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:49.815831 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:49.819283 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:49.820430 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:49.820484 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:49.820508 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.823985 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:49.824981 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:19:49.825001 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.829195 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:49.830106 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:19:49.830137 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.853602 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:49.854933 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:19:49.854955 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:49.862356 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:49.863754 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:19:49.863889 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:49 PP1 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:56 PP0 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:56 PP1 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:57 PP1 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP1 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP0 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:58 PP0 TP2 EP2] sglang is using nccl==2.30.7 +[2026-08-31 08:19:58 PP0 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:58 PP0 TP3 EP3] sglang is using nccl==2.30.7 +[2026-08-31 08:19:58 PP0 TP1 EP1] sglang is using nccl==2.30.7 +[2026-08-31 08:19:58 PP0 TP0 EP0] Init torch distributed ends. elapsed=9.24 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP1 TP3 EP3] Init torch distributed ends. elapsed=13.79 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP1 TP2 EP2] Init torch distributed ends. elapsed=9.38 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP1 TP1 EP1] Init torch distributed ends. elapsed=10.26 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP1 TP0 EP0] Init torch distributed ends. elapsed=8.67 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP0 TP3 EP3] Init torch distributed ends. elapsed=10.21 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP0 TP2 EP2] Init torch distributed ends. elapsed=13.70 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP0 TP1 EP1] Init torch distributed ends. elapsed=9.28 s, mem usage=0.50 GB +[2026-08-31 08:19:59 PP0 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP0 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP0 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP0 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP1 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP0 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP1 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP1 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP0 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP1 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP0 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP1 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP0 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP1 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP0 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP1 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP0 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP1 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP1 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP1 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP0 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP1 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP0 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP1 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP1 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP0 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP1 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP0 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP0 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP1 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP1 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP0 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). + Multi-thread loading shards: 0% Completed | 0/96 [00:00, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP1 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP1 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP0 TP1 EP1] DFLASH PP rank 0: capture layers=[], projection columns=[], incoming=False +[2026-08-31 08:21:20 PP0 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP1 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP1 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP1 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP1 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP0 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP0 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP0 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP0 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP1 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP1 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP1 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP0 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP0 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP1 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP0 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP0 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:21:20 PP0 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:21:20 PP0 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:21:20 PP0 TP0 EP0] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:20 PP0 TP3 EP3] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:20 PP0 TP2 EP2] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:20 PP1 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP1 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP0 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:21:20 PP1 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP1 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP1 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:20 PP1 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:20 PP0 TP1 EP1] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:20 PP1 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:20 PP1 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP1 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP1 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP0 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP0 TP0 EP0] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:22 PP1 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP1 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP0 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP0 TP2 EP2] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:22 PP0 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP0 TP1 EP1] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:22 PP0 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP0 TP3 EP3] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:21:22 PP1 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP1 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP1 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP1 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:23 PP0 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:23 PP0 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:23 PP0 TP1 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp0_dp0.json +[2026-08-31 08:21:23 PP0 TP3 EP3] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp0_dp0.json +[2026-08-31 08:21:23 PP0 TP2 EP2] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp0_dp0.json +[2026-08-31 08:21:23 PP0 TP0 EP0] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp0_dp0.json +[2026-08-31 08:21:23 PP0 TP2 EP2] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23 PP0 TP3 EP3] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23 PP1 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:23 PP1 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:23 PP0 TP1 EP1] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=False, pid=1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_1.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_1.log new file mode 100644 index 0000000..60623b6 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_1.log @@ -0,0 +1,628 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 08:18:58] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:18:58] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:18:58] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:19:01] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/root/.cache/sglang). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:19:01] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 1, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 270427961, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:27 PP3 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:27 PP3 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:30 PP2 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:30 PP2 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:31 PP2 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:31 PP2 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:32 PP3 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:32 PP3 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:32 PP3 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:32 PP3 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:32.469815 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:32.469883 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:32.469895 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:32.469926 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15755 +I0831 08:19:32.470013 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +[2026-08-31 08:19:32 PP2 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:32 PP2 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0831 08:19:32.493834 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:32.499552 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:32.505760 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:32.512733 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:32.515441 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:32.515538 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:32.515595 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:32.522768 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:32.524343 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:32.524374 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:32.547991 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:32.549387 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:32.549417 231 rdma_context.cpp:207] Using SIEVE endpoint store +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +I0831 08:19:32.576007 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:32.577409 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:32.577435 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:32.604003 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:32.605383 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:32.605547 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:32 PP3 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:32 PP3 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:32 PP3 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:33 PP2 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP2 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:33 PP2 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:33 PP2 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:33 PP3 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP3 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:33.732111 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:33.732182 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:33.732194 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:33.732226 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16864 +I0831 08:19:33.732321 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:33.758656 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:33.767715 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:33.776723 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:33.785722 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:33.789390 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:33.789491 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:33.789564 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:33.797785 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:33.799191 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:33.799219 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:33.824028 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:33.825368 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:33.825402 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:33.851992 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:33.853356 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:33.853380 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:33.879993 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:33.881299 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:33.881424 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:33 PP2 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:19:34 PP2 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:34 PP2 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:34.833853 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:34.833910 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:34.833920 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:34.833945 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15369 +I0831 08:19:34.834015 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:34.857606 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:34.862201 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:34.867720 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:34.873240 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:34.875504 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:34.875582 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:34.875626 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:34.881652 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:34.882902 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:34.882925 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:34.907984 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:34.909370 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:34.909399 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:34.936085 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:34.937490 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:34.937523 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:34.964035 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:34.965418 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:34.965552 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:34 PP2 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:19:36 PP3 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:36 PP3 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:36 PP2 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:36 PP2 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:36.572286 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:36.572347 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:36.572357 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:36.572383 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16551 +I0831 08:19:36.572482 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:36.593608 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:36.598219 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:36.603750 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:36.609256 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:36.611665 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:36.611750 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:36.611800 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.618389 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:36.619796 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:36.619866 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.644070 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:36.645426 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:36.645449 232 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:36.657166 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:36.657243 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:36.657254 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:36.657281 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15388 +I0831 08:19:36.657374 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:36.661943 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:36.666292 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:36.670565 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:36.671969 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:36.673364 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:36.673388 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.675424 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:36.677275 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:36.677341 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:36.677380 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.682632 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:36.683796 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:36.683818 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.700052 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:36.701510 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:36.701638 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:36 PP3 TP1 EP1] Init torch distributed begin. +I0831 08:19:36.708063 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:36.709416 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:36.709463 228 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 08:19:36 PP3 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:36 PP3 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:19:36.736094 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:36.737471 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:36.737509 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.764001 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:36.765368 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:36.765516 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:36 PP2 TP1 EP1] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:36.894125 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:36.894188 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:36.894197 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:36.894220 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16890 +I0831 08:19:36.894307 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:36.898207 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:36.901981 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:36.905854 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:36.931185 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:36.933578 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:36.933665 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:36.933712 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.940757 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:36.942183 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:36.942209 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.967965 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:36.969342 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:36.969374 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.996038 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:36.997459 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:36.997483 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.024099 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:37.025468 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:37.025594 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:37 PP3 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:19:37 PP2 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:37 PP2 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:37.820669 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:37.820729 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:37.820737 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:37.820758 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15258 +I0831 08:19:37.820843 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:37.846593 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:37.853714 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:37.860673 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:37.867650 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:37.870323 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:37.870416 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:37.870472 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.877671 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:37.879180 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:37.879209 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.903959 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:37.905351 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:37.905385 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.932013 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:37.933410 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:37.933434 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.960032 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:37.961469 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:37.961596 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:37 PP2 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:38 PP3 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:38 PP3 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.197093 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.197150 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.197160 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:19:38.197181 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16828 +I0831 08:19:38.197259 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.222533 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.228859 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:38.235872 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:38.242859 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.245533 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.245625 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.245679 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.252904 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.254350 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:19:38.254379 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.280050 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.281436 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:19:38.281466 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.308061 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.309466 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:19:38.309490 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.336009 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.337394 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:19:38.337522 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP3 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:19:56 PP3 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:56 PP2 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:57 PP3 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP3 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP2 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:58 PP3 TP3 EP3] Init torch distributed ends. elapsed=21.50 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP3 TP2 EP2] Init torch distributed ends. elapsed=20.18 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP3 TP1 EP1] Init torch distributed ends. elapsed=21.82 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP2 TP3 EP3] Init torch distributed ends. elapsed=23.56 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP3 TP0 EP0] Init torch distributed ends. elapsed=25.92 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP2 TP2 EP2] Init torch distributed ends. elapsed=24.64 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP2 TP1 EP1] Init torch distributed ends. elapsed=21.76 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP2 TP0 EP0] Init torch distributed ends. elapsed=20.56 s, mem usage=0.58 GB +[2026-08-31 08:19:59 PP3 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP2 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP3 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP3 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP2 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP2 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP3 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP2 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP3 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP3 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP3 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP2 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP2 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP3 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP2 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP3 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP2 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP2 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP3 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP3 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP2 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP2 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP3 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP2 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP3 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP2 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP2 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP3 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP3 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP3 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP2 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP2 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:00 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:01 PP2 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:01 PP2 TP0 EP0] Load weight end. elapsed=62.61 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:21:04 PP3 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:04 PP3 TP3 EP3] Load weight end. elapsed=64.89 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:04 PP2 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:04 PP2 TP3 EP3] Load weight end. elapsed=65.27 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:21:04 PP3 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:04 PP3 TP2 EP2] Load weight end. elapsed=65.36 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:05 PP2 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:05 PP2 TP2 EP2] Load weight end. elapsed=66.35 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:21:05 PP3 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:05 PP3 TP0 EP0] Load weight end. elapsed=66.54 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:09 PP3 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:09 PP3 TP1 EP1] Load weight end. elapsed=70.65 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:09 PP2 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:09 PP2 TP1 EP1] Load weight end. elapsed=70.71 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:21:10 PP3 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP3 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP3 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP3 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP2 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP2 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP3 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP3 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP3 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP3 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP2 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP2 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP2 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP2 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:10 PP2 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:10 PP2 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:12 PP2 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:21:12 PP2 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP2 TP1 EP1] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:21:12 PP2 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:21:12 PP2 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP2 TP0 EP0] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:21:12 PP2 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:21:12 PP2 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP2 TP2 EP2] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:21:12 PP2 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:21:12 PP2 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP2 TP3 EP3] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:21:15 PP3 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:21:15 PP3 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP3 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:15 PP3 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:21:15 PP3 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP3 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:15 PP3 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:21:15 PP3 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP3 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:15 PP3 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:21:15 PP3 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP3 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:18 PP3 TP0 EP0] Load weight end. elapsed=3.05 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP3 TP1 EP1] Load weight end. elapsed=3.80 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP3 TP3 EP3] Load weight end. elapsed=3.68 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP2 TP2 EP2] Load weight end. elapsed=6.38 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP2 TP1 EP1] Load weight end. elapsed=6.43 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP3 TP2 EP2] Load weight end. elapsed=3.71 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP2 TP0 EP0] Load weight end. elapsed=6.42 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP2 TP3 EP3] Load weight end. elapsed=6.40 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP3 TP1 EP1] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:21:19 PP3 TP2 EP2] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:21:19 PP2 TP1 EP1] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:21:19 PP2 TP0 EP0] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:21:19 PP2 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP2 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP3 TP3 EP3] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:21:19 PP3 TP0 EP0] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:21:19 PP3 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP3 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP2 TP2 EP2] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:21:19 PP2 TP3 EP3] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:21:20 PP3 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP3 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP3 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP2 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP3 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP2 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP2 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP2 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP3 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP3 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP2 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP2 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP3 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP2 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP2 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP3 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP3 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP3 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP3 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP2 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP2 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP2 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP3 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP3 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP3 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP2 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:20 PP2 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:20 PP2 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:20 PP2 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP3 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP3 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP2 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP3 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP3 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP3 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP3 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP2 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP2 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP3 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP3 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP3 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP3 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP2 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP2 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP2 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP2 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:22 PP2 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP2 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:21:23 PP2 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:23 PP2 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:23 PP3 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:23 PP3 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:23 PP3 TP0 EP0] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp3_dp0.json +[2026-08-31 08:21:23 PP3 TP1 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp3_dp0.json +[2026-08-31 08:21:23 PP3 TP2 EP2] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp3_dp0.json +[2026-08-31 08:21:23 PP3 TP3 EP3] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp3_dp0.json +[2026-08-31 08:21:23 PP3 TP3 EP3] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23 PP3 TP0 EP0] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23 PP3 TP1 EP1] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23 PP3 TP2 EP2] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=False, pid=1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_2.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_2.log new file mode 100644 index 0000000..b9488e3 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_2.log @@ -0,0 +1,553 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 08:18:58] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:18:58] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:18:58] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:19:01] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/root/.cache/sglang). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:19:01] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 2, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 437065559, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:01] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:26 PP5 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:26 PP5 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:31 PP5 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:31 PP5 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:31.314847 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:31.314908 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:31.314919 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:31.314949 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16060 +I0831 08:19:31.315029 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:31.338516 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:31.343145 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:31.348732 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:31.354305 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:31.356760 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:31.356850 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:31.356904 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:31.363461 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:31.364888 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:31.364917 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:31.388852 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:31.390237 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:31.390261 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:31.416873 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:31.418265 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:31.418290 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:31.444881 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:31.446271 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:31.446410 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:31 PP5 TP1 EP1] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:32 PP4 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:32 PP4 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:33 PP5 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP5 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:33 PP4 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP4 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:33 PP4 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP4 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:34 PP5 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:34 PP5 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:34 PP4 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:34 PP4 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:34 PP5 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:34 PP5 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:36 PP4 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:36 PP4 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:36.974062 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:36.974129 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:36.974139 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:36.974169 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15143 +I0831 08:19:36.974272 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:36.998509 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:37.006013 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:37.014897 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:37.023877 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:37.027616 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:37.027709 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:37.027766 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.035905 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:37.037406 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:37.037436 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.064881 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:37.066285 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:37.066309 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.074498 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:37.075856 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:37.075879 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.100883 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:37.102308 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:37.102447 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:37 PP4 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:37 PP4 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:37 PP4 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:37.723565 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:37.723646 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:37.723658 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:37.723690 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15887 +I0831 08:19:37.723784 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:37.748377 229 topology.cpp:127] Device mlx5_0 port 1 is available +[2026-08-31 08:19:37 PP4 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:37 PP4 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:19:37.772764 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:37.796608 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:37.820667 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:37.823390 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:37.823483 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:37.823539 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.830654 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:37.832135 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:37.832163 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.856837 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:37.858230 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:37.858259 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.884936 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:37.886346 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:37.886380 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.912941 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:37.914341 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:37.914479 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:37 PP4 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:19:37 PP5 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:37 PP5 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.035291 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.035387 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.035403 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:38.035439 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15212 +I0831 08:19:38.035553 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.060900 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.068013 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:38.074960 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:38.081894 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.084563 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.084661 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.084715 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.091848 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.093305 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:38.093334 230 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.104502 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.104553 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.104563 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:38.104583 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16011 +I0831 08:19:38.104686 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.116828 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.118175 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:38.118203 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.125597 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.130121 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:38.144836 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.146205 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:38.146229 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.156898 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:38.172920 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.174300 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:38.174427 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP4 TP3 EP3] Init torch distributed begin. +I0831 08:19:38.180706 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.183472 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.183573 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.183632 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.209002 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.210440 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:38.210474 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.217823 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.219192 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:38.219218 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.226564 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.227921 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:38.227945 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.252893 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.254210 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:38.254330 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP5 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:19:38 PP4 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:38 PP4 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:38 PP5 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:38 PP5 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.663432 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.663499 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.663509 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:38.663532 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16406 +I0831 08:19:38.663618 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.686415 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.712720 234 topology.cpp:127] Device mlx5_1 port 1 is available +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.717761 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.717839 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.717850 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:38.717876 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16668 +I0831 08:19:38.717974 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.719749 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:38.722591 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.726703 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.729362 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.729466 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.729537 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.736645 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.738123 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:38.738265 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.748168 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:38.755101 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:38.761986 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.764627 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.764721 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.764775 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.764856 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.766234 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:38.766299 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.771967 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.773401 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:38.773429 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.773633 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.774991 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:38.775033 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.797140 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.798763 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:38.798802 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.800904 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.802255 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:38.802384 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP5 TP3 EP3] Init torch distributed begin. +I0831 08:19:38.829849 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.831151 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:38.831174 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.860962 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.862143 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:38.862233 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP4 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:19:39 PP5 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:39 PP5 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:39.305204 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:39.305266 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:39.305276 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:19:39.305300 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15909 +I0831 08:19:39.305370 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:39.309270 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:39.330283 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:39.335806 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:39.341331 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:39.343722 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:39.343808 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:39.343856 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.350394 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:39.351827 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:19:39.351855 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.359263 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:39.360615 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:19:39.360641 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.368054 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:39.369426 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:19:39.369450 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.376860 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:39.378204 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:19:39.378320 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:39 PP5 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:56 PP4 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:56 PP5 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:57 PP5 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP5 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP4 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:58 PP5 TP3 EP3] Init torch distributed ends. elapsed=19.71 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP5 TP1 EP1] Init torch distributed ends. elapsed=27.07 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP5 TP0 EP0] Init torch distributed ends. elapsed=19.14 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP5 TP2 EP2] Init torch distributed ends. elapsed=20.26 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP4 TP3 EP3] Init torch distributed ends. elapsed=20.34 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP4 TP1 EP1] Init torch distributed ends. elapsed=19.65 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP4 TP0 EP0] Init torch distributed ends. elapsed=21.41 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP4 TP2 EP2] Init torch distributed ends. elapsed=20.60 s, mem usage=0.58 GB +[2026-08-31 08:19:59 PP5 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP5 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP5 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP5 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP4 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP4 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP5 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP4 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP4 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP5 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP4 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP4 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP5 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP4 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP5 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP4 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP5 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP4 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP5 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP4 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP5 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP4 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP4 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP5 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP4 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP5 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP4 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP5 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP4 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP4 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP5 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP5 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:04 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:05 PP4 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:05 PP4 TP0 EP0] Load weight end. elapsed=66.26 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:06 PP4 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:06 PP4 TP2 EP2] Load weight end. elapsed=67.17 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:06 PP4 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:06 PP4 TP3 EP3] Load weight end. elapsed=67.34 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:06 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:07 PP5 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:07 PP5 TP0 EP0] Load weight end. elapsed=68.67 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:07 PP4 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:07 PP4 TP1 EP1] Load weight end. elapsed=68.74 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:08 PP4 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:08 PP4 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:08 PP4 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:08 PP4 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:08 PP4 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:08 PP4 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:08 PP4 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:08 PP4 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:08 PP5 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:08 PP5 TP1 EP1] Load weight end. elapsed=69.72 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:10 PP5 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:10 PP5 TP3 EP3] Load weight end. elapsed=70.93 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:11 PP5 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:11 PP5 TP2 EP2] Load weight end. elapsed=72.33 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:11 PP4 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:21:11 PP4 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:11 PP4 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:11 PP4 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:21:11 PP4 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:11 PP4 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:21:11 PP4 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:11 PP4 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:11 PP4 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:11 PP4 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:21:11 PP4 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:11 PP4 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:11 PP5 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:11 PP5 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:11 PP5 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:11 PP5 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:11 PP5 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:11 PP5 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:11 PP5 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:11 PP5 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:12 PP5 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:21:12 PP5 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP5 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:12 PP5 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:21:12 PP5 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP5 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:12 PP5 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:21:12 PP5 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP5 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:12 PP5 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:21:12 PP5 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP5 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:17 PP5 TP2 EP2] Load weight end. elapsed=5.89 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP4 TP1 EP1] Load weight end. elapsed=7.19 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP4 TP0 EP0] Load weight end. elapsed=7.20 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP5 TP3 EP3] Load weight end. elapsed=6.61 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP5 TP1 EP1] Load weight end. elapsed=6.63 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP4 TP3 EP3] Load weight end. elapsed=7.19 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP5 TP0 EP0] Load weight end. elapsed=6.63 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP4 TP2 EP2] Load weight end. elapsed=7.20 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:19 PP5 TP1 EP1] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:21:19 PP5 TP2 EP2] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:21:19 PP5 TP0 EP0] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:21:19 PP4 TP0 EP0] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:21:19 PP5 TP3 EP3] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:21:19 PP5 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP5 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP4 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP4 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP4 TP2 EP2] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:21:19 PP4 TP1 EP1] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:21:19 PP4 TP3 EP3] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:21:20 PP5 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP5 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP5 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP5 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP4 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP4 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP4 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP4 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP5 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP5 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP4 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP4 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP4 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP5 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP5 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP4 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP5 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP5 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP5 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP4 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP5 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP4 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP4 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP4 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP5 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP5 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP4 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP4 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP4 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP5 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP5 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP4 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP4 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP4 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP5 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP5 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP4 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:22 PP4 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:22 PP5 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP5 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP4 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP4 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP4 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP4 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP5 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP4 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP5 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP4 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP5 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP5 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:23 PP5 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:23 PP5 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:23 PP5 TP2 EP2] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp5_dp0.json +[2026-08-31 08:21:23 PP5 TP0 EP0] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp5_dp0.json +[2026-08-31 08:21:23 PP5 TP1 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp5_dp0.json +[2026-08-31 08:21:23 PP5 TP3 EP3] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp5_dp0.json +[2026-08-31 08:21:23 PP4 TP0 EP0] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp4_dp0.json +[2026-08-31 08:21:23 PP4 TP1 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp4_dp0.json +[2026-08-31 08:21:23 PP4 TP2 EP2] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp4_dp0.json +[2026-08-31 08:21:23 PP4 TP3 EP3] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp4_dp0.json +[2026-08-31 08:21:23 PP5 TP1 EP1] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23 PP5 TP3 EP3] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=False, pid=1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_3.log new file mode 100644 index 0000000..80a6ac3 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/p_3.log @@ -0,0 +1,550 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 08:18:59] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:18:59] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:18:59] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:19:02] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/root/.cache/sglang). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:19:02] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 3, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 1059999035, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:19:02] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:29 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:29 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:32 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:32 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:33 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:33 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:33 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:34 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:34 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:34 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:34 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:34 PP7 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:34 PP7 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:34 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:34 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:34.986764 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:34.986845 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:34.986855 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:34.986886 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16291 +I0831 08:19:34.986953 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:35.008636 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:35.013226 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:35.017715 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:35.023200 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:35.025238 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:35.025317 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:35.025363 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:35.031092 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:35.032358 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:35.032383 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:35.059674 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:35.061120 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:35.061161 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:35.087539 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:35.088948 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:35.088974 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:35.115535 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:35.116916 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:35.117092 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:35 PP7 TP0 EP0] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:19:35 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:19:35 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:19:35 PP6 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:35 PP6 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:36.134233 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:36.134341 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:36.134352 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:36.134388 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15701 +I0831 08:19:36.134479 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:36.158968 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:36.167981 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:36.176939 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:36.185909 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:36.189558 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:36.189649 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:36.189706 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.197832 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:36.199224 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:36.199251 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.227886 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:36.229355 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:36.229396 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.255549 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:36.256894 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:36.256920 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:36.283820 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:36.285279 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:36.285398 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:36 PP6 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:19:36 PP7 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:36 PP7 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:37 PP6 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:37 PP6 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:37.139938 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:37.140040 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:37.140055 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:37.140096 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15255 +I0831 08:19:37.140215 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:37.146139 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:37.152051 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:37.158051 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:37.164471 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:37.167126 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:37.167212 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:37.167263 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.173868 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:37.175171 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:37.175196 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.199565 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:37.200943 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:37.200969 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.227507 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:37.228852 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:37.228874 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.255460 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:37.256250 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:37.256322 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:37.256332 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:37.256359 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16183 +I0831 08:19:37.256481 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:37.256804 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:37.256920 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:37 PP7 TP2 EP2] Init torch distributed begin. +I0831 08:19:37.282440 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:37.288795 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:37.295730 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:37.302639 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:37.305285 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:37.305377 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:37.305433 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.312532 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:37.314009 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:37.314043 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.339496 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:37.340853 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:37.340884 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.367480 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:37.368839 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:37.368866 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:37.395510 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:37.396881 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:37.397011 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:37 PP6 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:19:38 PP6 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:38 PP6 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:19:38 PP7 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:38 PP7 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.727011 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.727135 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.727154 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:38.727198 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15909 +I0831 08:19:38.727319 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.751577 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.775381 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:38.782401 230 topology.cpp:127] Device mlx5_2 port 1 is available +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:38.786558 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:38.786612 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:38.786620 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:38.786640 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16831 +I0831 08:19:38.786701 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:38.790109 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:38.793380 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:38.796545 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:38.799685 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.800863 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.800911 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.800938 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.804558 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.805393 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:38.805406 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.807274 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:38.809918 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:38.810017 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:38.810084 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.817189 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:38.818648 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:38.818678 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.830020 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.831192 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:38.831216 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.843428 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:38.844803 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:38.844830 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.855489 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.856848 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:38.856871 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.871500 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:38.872892 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:38.872920 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:38.883467 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.884812 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:38.884930 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP7 TP1 EP1] Init torch distributed begin. +I0831 08:19:38.899626 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:38.901074 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:38.901223 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:38 PP6 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:19:38 PP7 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:38 PP7 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:39.139494 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:39.139556 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:39.139564 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:39.139587 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15138 +I0831 08:19:39.139668 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:39.143599 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:39.147344 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:39.150959 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:39.173837 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:39.176290 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:39.176383 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:39.176437 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.183485 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:39.184929 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:39.184957 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.211467 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:39.212850 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:39.212874 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.239450 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:39.240860 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:39.240885 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.267432 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:39.268800 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:39.268920 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:39 PP7 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:19:39 PP6 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:19:39 PP6 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:19:39.806905 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:19:39.806960 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:19:39.806968 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:19:39.806988 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15401 +I0831 08:19:39.807067 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:19:39.833032 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:19:39.863338 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:19:39.870296 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:19:39.877213 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:19:39.879853 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:19:39.879947 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:19:39.880002 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.887115 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:19:39.888533 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:19:39.888562 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.915402 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:19:39.916739 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:19:39.916765 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.943514 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:19:39.944882 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:19:39.944908 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:19:39.971626 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:19:39.973037 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:19:39.973162 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:19:39 PP6 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:19:56 PP6 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:56 PP7 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:19:57 PP6 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP6 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:57 PP7 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:19:58 PP7 TP3 EP3] Init torch distributed ends. elapsed=19.29 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP7 TP1 EP1] Init torch distributed ends. elapsed=19.68 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP7 TP2 EP2] Init torch distributed ends. elapsed=21.31 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP7 TP0 EP0] Init torch distributed ends. elapsed=23.45 s, mem usage=0.50 GB +[2026-08-31 08:19:58 PP6 TP3 EP3] Init torch distributed ends. elapsed=19.66 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP6 TP2 EP2] Init torch distributed ends. elapsed=21.17 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP6 TP1 EP1] Init torch distributed ends. elapsed=22.28 s, mem usage=0.58 GB +[2026-08-31 08:19:58 PP6 TP0 EP0] Init torch distributed ends. elapsed=18.59 s, mem usage=0.58 GB +[2026-08-31 08:19:59 PP7 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP7 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP7 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP7 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP6 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP6 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP6 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP6 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP6 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP6 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP7 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP7 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP7 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP7 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP6 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP6 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP6 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP6 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP7 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP6 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP6 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP7 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP7 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:19:59 PP7 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP7 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP7 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP6 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:19:59 PP6 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:19:59 PP7 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP7 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:19:59 PP6 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:19:59 PP6 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:21:08 PP7 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:08 PP7 TP3 EP3] Load weight end. elapsed=69.42 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:21:08 PP6 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:08 PP6 TP3 EP3] Load weight end. elapsed=69.41 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:09 PP7 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:09 PP7 TP2 EP2] Load weight end. elapsed=70.09 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:21:10 PP6 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:10 PP6 TP1 EP1] Load weight end. elapsed=71.18 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:11 PP6 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:11 PP6 TP2 EP2] Load weight end. elapsed=72.72 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:12 PP7 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:12 PP7 TP1 EP1] Load weight end. elapsed=72.85 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:21:12 PP6 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:12 PP6 TP0 EP0] Load weight end. elapsed=73.02 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:21:12 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:12 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:12 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:12 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:12 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:12 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:12 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:12 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:12 PP6 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:21:12 PP6 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP6 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:12 PP6 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:21:12 PP6 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP6 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:12 PP6 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:21:12 PP6 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP6 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:12 PP6 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:21:12 PP6 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:12 PP6 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:21:14 PP7 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:21:14 PP7 TP0 EP0] Load weight end. elapsed=75.67 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:21:15 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:15 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:15 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:15 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:15 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:15 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:15 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:21:15 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:21:15 PP7 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:21:15 PP7 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP7 TP0 EP0] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:21:15 PP7 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:21:15 PP7 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP7 TP1 EP1] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:21:15 PP7 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:21:15 PP7 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP7 TP2 EP2] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:21:15 PP7 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:21:15 PP7 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:21:15 PP7 TP3 EP3] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:21:17 PP6 TP0 EP0] Load weight end. elapsed=4.68 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP7 TP1 EP1] Load weight end. elapsed=2.58 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:21:18 PP6 TP3 EP3] Load weight end. elapsed=5.17 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP6 TP2 EP2] Load weight end. elapsed=5.17 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP7 TP3 EP3] Load weight end. elapsed=2.48 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:21:18 PP7 TP0 EP0] Load weight end. elapsed=2.61 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:21:18 PP6 TP1 EP1] Load weight end. elapsed=5.20 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:21:18 PP7 TP2 EP2] Load weight end. elapsed=2.48 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:21:19 PP6 TP0 EP0] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:21:19 PP7 TP1 EP1] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:21:19 PP6 TP1 EP1] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:21:19 PP7 TP0 EP0] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:21:19 PP6 TP2 EP2] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:21:19 PP7 TP3 EP3] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:21:19 PP7 TP2 EP2] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:21:19 PP6 TP3 EP3] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:21:19 PP6 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP6 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:19 PP7 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:21:19 PP7 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:21:20 PP7 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP7 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP7 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP7 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP6 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP6 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP6 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP6 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:21:20 PP6 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP7 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP6 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP7 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP6 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP6 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:21:20 PP7 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP7 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:21:20 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP6 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP6 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP6 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:21:20 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:21:20 PP6 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:20 PP7 TP3 EP3] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:21:20 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:21:20 PP7 TP2 EP2] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:21:20 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:21:20 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:21:20 PP7 TP0 EP0] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:21:20 PP7 TP1 EP1] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:21:22 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:21:22 PP7 TP1 EP1] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:21:22 PP7 TP1 EP1] Overriding draft attention backend to triton. +[2026-08-31 08:21:22 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP6 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP6 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP6 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:21:22 PP6 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:21:22 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:21:22 PP7 TP2 EP2] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:21:22 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:21:22 PP7 TP0 EP0] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:21:22 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:21:22 PP7 TP3 EP3] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:21:23 PP6 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:21:23 PP6 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:21:23 PP6 TP0 EP0] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp6_dp0.json +[2026-08-31 08:21:23 PP6 TP2 EP2] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp6_dp0.json +[2026-08-31 08:21:23 PP6 TP3 EP3] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp6_dp0.json +[2026-08-31 08:21:23 PP6 TP1 EP1] Running FlashInfer autotune with cache: /root/.cache/sglang/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp6_dp0.json +[2026-08-31 08:21:23 PP7 TP2 EP2] Overriding draft attention backend to triton. +[2026-08-31 08:21:23 PP6 TP2 EP2] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23 PP6 TP3 EP3] Scheduler hit an exception: Traceback (most recent call last): + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 5428, in run_scheduler_process + scheduler = Scheduler( + ^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 541, in __init__ + self.init_model_worker() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1048, in init_model_worker + self.init_all_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/scheduler.py", line 1031, in init_all_cuda_graphs + self.tp_worker.init_cuda_graphs() + File "/opt/kimi-dflash/python/sglang/srt/managers/tp_worker.py", line 443, in init_cuda_graphs + self.model_runner.init_cuda_graphs( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner.py", line 1048, in init_cuda_graphs + capture = capture_cuda_graphs( + ^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py", line 178, in capture_cuda_graphs + eager_runner = EagerRunner(model_runner) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/eager_runner.py", line 155, in __init__ + self.warmup() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 260, in warmup + self._flashinfer_autotune(buffers=buffers, batch_size=batch_size) + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 330, in _flashinfer_autotune + run_flashinfer_autotune_forward( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/flashinfer_autotune.py", line 293, in run_flashinfer_autotune_forward + forward_fn() + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 324, in forward_fn + self._dummy_run( + File "/opt/kimi-dflash/python/sglang/srt/model_executor/runner/base_runner.py", line 639, in _dummy_run + mr.attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py", line 1075, in init_forward_metadata + attn_backend.init_forward_metadata(forward_batch) + File "/opt/kimi-dflash/python/sglang/srt/layers/attention/triton_backend.py", line 808, in init_forward_metadata + if spec_info is None or spec_info.kv_indptr is None: + ^^^^^^^^^^^^^^^^^^^ +AttributeError: 'DFlashVerifyInput' object has no attribute 'kv_indptr' + +[2026-08-31 08:21:23] Received sigquit from a child process. It usually means the child failed. +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=True, pid=1 +[2026-08-31 08:21:23] kill_process_tree called: parent_pid=1, include_parent=False, pid=1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/router.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/router.log new file mode 100644 index 0000000..4ac37f7 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/logs/router.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_router_0 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/metadata/preflight.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/metadata/preflight.txt new file mode 100644 index 0000000..5825cc6 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-20260831-1617/metadata/preflight.txt @@ -0,0 +1,8 @@ +174.1.60.1 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.2 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.3 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.4 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.5 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.6 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.7 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 +174.1.60.8 sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_0.cmd.txt new file mode 100644 index 0000000..fc2f560 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.1 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_1.cmd.txt new file mode 100644 index 0000000..256cf64 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.2 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_2.cmd.txt new file mode 100644 index 0000000..2dadb28 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.3 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_3.cmd.txt new file mode 100644 index 0000000..15d0822 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/commands/p_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_validation_p_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -e SGLANG_HOST_IP=174.1.60.4 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-pwarm1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --speculative-draft-kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.88 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 --enable-metrics --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/exit_code b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/exit_code new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/exit_code @@ -0,0 +1 @@ +1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_0.log new file mode 100644 index 0000000..8a2955b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_0.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_0 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_1.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_1.log new file mode 100644 index 0000000..f647e30 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_1.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_1 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_2.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_2.log new file mode 100644 index 0000000..c1ed8e8 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_2.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_2 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_3.log new file mode 100644 index 0000000..857ef64 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/d_3.log @@ -0,0 +1 @@ +Error response from daemon: No such container: kimi3_pro6000_pd_dflash_validation_d_3 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_0.log new file mode 100644 index 0000000..6859cb9 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-pwarm1-20260831-1652/logs/p_0.log @@ -0,0 +1,582 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 08:53:15] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:53:15] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:53:15] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:53:18] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:53:18] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 0, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 431940275, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:23] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:23] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:24] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:24] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:24] CommonKVBootstrapServer started successfully on 0.0.0.0:28800 +[2026-08-31 08:53:24] No HuggingFace chat template found +[2026-08-31 08:53:24] No chat template found, defaulting to 'string' content format +[2026-08-31 08:53:44] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:44] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +/opt/kimi-dflash/python/sglang/srt/utils/hf_transformers/tokenizer.py:433: UserWarning: Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead. + warnings.warn( +[2026-08-31 08:53:45] Applying special tokens cache patch for Kimi tokenizer: +[2026-08-31 08:53:45 PP1 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:45 PP1 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:47 PP0 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:47 PP0 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:47 PP0 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:47 PP0 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:47 PP0 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:47 PP0 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:48 PP1 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:48 PP1 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:48 PP1 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:48 PP1 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:48 PP1 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:48 PP1 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:48.851246 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:48.851315 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:48.851325 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:48.851353 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15754 +I0831 08:53:48.851420 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:48.878397 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:48.885684 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:48.909976 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:48.917057 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:48.919821 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:48.919932 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:48.919993 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:48.927202 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:48.928665 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:48.928695 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:48.954566 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:48.956091 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:48.956128 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:48.982297 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:48.983769 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:48.983798 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:49.010238 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:49.011662 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:49.011802 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:49 PP1 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:53:49 PP0 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:49 PP0 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:49 PP1 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:49 PP1 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:50 PP0 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:50 PP0 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:50 PP0 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:50 PP0 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:51 PP0 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:51 PP0 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:51.218081 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:51.218190 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:51.218209 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:51.218248 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15339 +I0831 08:53:51.218379 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:51.246188 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:51.255300 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:51.264262 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:51.273257 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:51.276938 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:51.277037 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:51.277094 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.302300 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:51.303754 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:51.303784 230 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:51.309075 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:51.309149 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:51.309162 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:51.309187 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15107 +I0831 08:53:51.309295 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:51.326697 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:51.326781 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:51.326793 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:51.326826 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16058 +I0831 08:53:51.326975 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:51.330256 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:51.331620 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:51.331645 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.331782 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:51.332705 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:51.337412 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:51.338378 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:51.346289 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:51.353209 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:51.355880 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:51.355974 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:51.356029 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.358213 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:51.359628 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:51.359653 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.361584 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:51.363173 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:51.364658 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:51.364688 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.368510 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:51.371170 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:51.371259 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:51.371312 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.386211 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:51.387554 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:51.387701 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:51 PP0 TP3 EP3] Init torch distributed begin. +I0831 08:53:51.390224 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:51.391649 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:51.391696 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.398502 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:51.399964 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:51.399999 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.418226 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:51.419586 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:51.419611 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.426187 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:51.426910 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:51.427553 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:51.427579 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.428251 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:51.428363 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:51 PP0 TP0 EP0] Init torch distributed begin. +I0831 08:53:51.454253 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:51.455626 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:51.455650 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:51.482226 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:51.483585 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:51.483712 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:51 PP0 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:53:52 PP1 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:52 PP1 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:52.427794 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:52.427870 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:52.427882 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:52.427911 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16409 +I0831 08:53:52.427997 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:52.452459 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:52.458747 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:52.465193 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:52.472150 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:52.474833 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:52.474931 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:52.474992 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:52.482293 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:52.483898 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:52.483929 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:52.510562 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:52.512024 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:52.512061 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:52.538281 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:52.539660 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:52.539696 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:52.566253 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:52.567642 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:52.567775 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:52 PP1 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:53:52 PP1 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:52 PP1 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:53.027081 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:53.027146 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:53.027158 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:53.027180 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16999 +I0831 08:53:53.027263 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +[2026-08-31 08:53:53 PP0 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:53 PP0 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:53:53.051783 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:53.057415 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:53.062992 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:53.069389 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:53.071887 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:53.071981 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:53.072036 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.079182 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:53.080660 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:53.080689 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.106209 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:53.107587 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:53.107612 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.134284 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:53.135648 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:53.135680 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.162200 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:53.163561 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:53.163697 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:53 PP1 TP1 EP1] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:53.234241 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:53.234304 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:53.234314 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:53.234338 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15189 +I0831 08:53:53.234414 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:53.256399 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:53.262089 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:53.268479 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:53.275384 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:53.278066 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:53.278160 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:53.278216 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.285328 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:53.286741 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:53.286769 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.314236 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:53.315599 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:53.315623 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.342191 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:53.343570 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:53.343595 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.370215 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:53.371579 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:53.371711 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:53 PP0 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:53:54 PP1 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:54 PP1 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:54.598055 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:54.598124 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:54.598132 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 08:53:54.598157 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15152 +I0831 08:53:54.598235 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:54.620482 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:54.626257 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:54.632704 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:54.639654 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:54.642341 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:54.642436 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:54.642489 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.649744 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:54.651183 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 08:53:54.651211 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.678174 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:54.679533 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 08:53:54.679569 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.706241 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:54.707608 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 08:53:54.707635 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.734228 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:54.735618 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 08:53:54.735759 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:54 PP1 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:54:02 PP0 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:02 PP1 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP0 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP1 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP0 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP0 TP3 EP3] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP0 TP1 EP1] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP0 TP2 EP2] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP0 TP0 EP0] Init torch distributed ends. elapsed=12.42 s, mem usage=0.50 GB +[2026-08-31 08:54:03 PP1 TP2 EP2] Init torch distributed ends. elapsed=9.12 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP1 TP3 EP3] Init torch distributed ends. elapsed=14.84 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP1 TP0 EP0] Init torch distributed ends. elapsed=11.28 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP1 TP1 EP1] Init torch distributed ends. elapsed=10.69 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP0 TP3 EP3] Init torch distributed ends. elapsed=12.46 s, mem usage=0.50 GB +[2026-08-31 08:54:03 PP0 TP1 EP1] Init torch distributed ends. elapsed=10.48 s, mem usage=0.50 GB +[2026-08-31 08:54:03 PP0 TP2 EP2] Init torch distributed ends. elapsed=12.37 s, mem usage=0.50 GB +[2026-08-31 08:54:04 PP0 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP0 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP0 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP0 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP1 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP1 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP1 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP0 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP1 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP1 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP1 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP0 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP0 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP0 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP1 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP0 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP0 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP1 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP1 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP1 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP1 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP0 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP1 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP1 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP1 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP0 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP0 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP0 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP0 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP0 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP1 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP1 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). + Multi-thread loading shards: 0% Completed | 0/96 [00:00, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:44 PP0 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:45 PP0 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP1 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP1 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP1 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP1 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP0 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP0 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP0 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP0 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP1 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP1 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP0 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP1 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP1 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP0 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP0 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP0 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:56:45 PP0 TP0 EP0] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP0 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:56:45 PP0 TP3 EP3] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP1 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP0 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:56:45 PP1 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP0 TP1 EP1] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP1 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP1 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP0 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 08:56:45 PP1 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP1 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP0 TP2 EP2] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP1 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP1 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP0 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP0 TP0 EP0] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP0 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP0 TP3 EP3] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP1 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP0 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP1 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP0 TP1 EP1] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP1 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP1 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP1 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP1 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP0 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP0 TP2 EP2] Memory pool end. avail mem=33.60 GB +[2026-08-31 08:56:45 PP1 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP1 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP0 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP0 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP1 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP1 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP1 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp1_dp0.json +[2026-08-31 08:56:45 PP1 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp1_dp0.json +[2026-08-31 08:56:45 PP1 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp1_dp0.json +[2026-08-31 08:56:45 PP1 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp1_dp0.json +[2026-08-31 08:56:45 PP0 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp0_dp0.json +[2026-08-31 08:56:45 PP0 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp0_dp0.json +[2026-08-31 08:56:45 PP0 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp0_dp0.json +[2026-08-31 08:56:45 PP0 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp0_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:00 [options] + warnings.warn( +[2026-08-31 08:53:18] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:53:18] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:53:18] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:53:21] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:53:21] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 1, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 602109307, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:45 PP3 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:45 PP3 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:48 PP3 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:48 PP3 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:49 PP3 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:49 PP3 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:49.315522 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:49.315586 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:49.315598 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:49.315629 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15981 +I0831 08:53:49.315717 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:49.337613 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:49.342199 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:49.367753 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:49.374650 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:49.377321 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:49.377414 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:49.377472 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:49.384552 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:49.386066 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:49.386097 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:49.411988 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:49.413380 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:49.413407 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:49.440024 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:49.441433 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:49.441459 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:49.468000 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:49.469399 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:49.469537 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:49 PP3 TP0 EP0] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:49 PP2 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:49 PP2 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:50 PP3 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP3 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:50 PP3 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP3 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:50 PP2 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP2 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:53 PP3 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:53 PP3 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:53 PP2 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:53 PP2 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:53 PP2 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:53 PP2 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:53 PP2 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:53 PP2 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:53.599541 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:53.599632 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:53.599646 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:53.599686 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16682 +I0831 08:53:53.599792 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:53.627486 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:53.636332 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:53.645241 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:53.654626 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:53.658301 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:53.658409 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:53.658476 232 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:53.669219 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:53.669297 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:53.669310 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:53.669343 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16360 +I0831 08:53:53.669487 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:53.675109 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:53.681407 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:53.684101 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:53.685568 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:53.685600 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.687625 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:53.692828 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:53.694033 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:53.695067 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:53.695144 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:53.695191 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.695425 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:53.695452 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.703792 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:53.705181 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:53.705206 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.720085 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:53.721611 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:53.721642 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.732045 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:53.733444 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:53.733625 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:53 PP3 TP1 EP1] Init torch distributed begin. +I0831 08:53:53.748067 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:53.749445 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:53.749478 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.776011 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +[2026-08-31 08:53:53 PP3 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:53 PP3 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:53:53.777431 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:53.777453 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:53.808048 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:53.809480 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:53.809631 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:53 PP2 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:53.942822 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:53.942876 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:53.942884 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:53.942903 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16614 +I0831 08:53:53.942971 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:53.965081 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:53.969337 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:53.974212 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:53.999732 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:54.002384 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:54.002477 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:54.002532 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.009578 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:54.010982 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:54.011011 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.035912 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:54.037254 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:54.037286 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.063997 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:54.065379 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:54.065404 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.092036 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:54.093412 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:54.093541 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:54 PP3 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:53:54 PP2 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:54 PP2 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:54.652940 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:54.653004 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:54.653013 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:54.653038 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15582 +I0831 08:53:54.653118 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:54.678263 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:54.684098 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:54.690465 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:54.697283 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:54.699922 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:54.700011 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:54.700063 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.707149 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:54.708678 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:54.708730 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.735925 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:54.737288 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:54.737314 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.764007 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:54.765415 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:54.765440 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.791989 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:54.793361 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:54.793485 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:54 PP2 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:53:54 PP3 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:54 PP3 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:55.173295 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:55.173357 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:55.173365 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:55.173386 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16849 +I0831 08:53:55.173476 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:55.177354 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:55.181114 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:55.184764 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:55.188808 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:55.190183 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:55.190237 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:55.190265 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.194392 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:55.195405 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:55.195423 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.219537 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:55.220896 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:55.220959 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.248030 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:55.249382 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:55.249406 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.275946 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:55.277312 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:55.277438 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:55 PP3 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:53:57 PP2 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:57 PP2 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:57 PP2 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:57 PP2 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:58.080351 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:58.080417 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:58.080427 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:58.080456 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16563 +I0831 08:53:58.080576 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:58.100751 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:58.100826 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:58.100837 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 08:53:58.100865 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15006 +I0831 08:53:58.100952 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:58.101724 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:58.105619 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:58.107399 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:58.110019 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:58.113221 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:58.114691 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:58.119449 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:58.119676 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:58.121794 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:58.121865 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:58.121903 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.122748 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:58.122844 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:58.122898 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.127197 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:58.128361 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:58.128381 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.130127 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:58.131685 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 08:53:58.131713 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.156121 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:58.156163 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:58.157658 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:58.157675 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 08:53:58.157706 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.157713 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.165056 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:58.166509 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:58.166533 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.184064 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:58.185468 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 08:53:58.185506 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.192337 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:58.193851 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:58.193984 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:58 PP2 TP1 EP1] Init torch distributed begin. +I0831 08:53:58.212090 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:58.213476 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 08:53:58.213614 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:58 PP2 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:54:02 PP2 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:02 PP3 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP2 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP2 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP3 TP3 EP3] Init torch distributed ends. elapsed=9.72 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP3 TP2 EP2] Init torch distributed ends. elapsed=8.54 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP3 TP1 EP1] Init torch distributed ends. elapsed=10.08 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP3 TP0 EP0] Init torch distributed ends. elapsed=14.35 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP2 TP3 EP3] Init torch distributed ends. elapsed=9.02 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP2 TP2 EP2] Init torch distributed ends. elapsed=10.01 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP2 TP1 EP1] Init torch distributed ends. elapsed=5.62 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP2 TP0 EP0] Init torch distributed ends. elapsed=5.60 s, mem usage=0.58 GB +[2026-08-31 08:54:04 PP2 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP3 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP3 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP3 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP2 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP2 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP2 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP3 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP3 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP3 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP3 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP2 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP2 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP3 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP2 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP2 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP2 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP3 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP2 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP3 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP3 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP3 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP2 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP2 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP3 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP3 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP2 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP3 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP2 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP3 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP2 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP2 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:06 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:56:40 PP3 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP3 TP0 EP0] Load weight end. elapsed=155.91 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP3 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP3 TP3 EP3] Load weight end. elapsed=156.05 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP2 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP2 TP2 EP2] Load weight end. elapsed=156.09 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:56:40 PP3 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP3 TP2 EP2] Load weight end. elapsed=156.15 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP2 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP2 TP1 EP1] Load weight end. elapsed=156.20 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:56:40 PP3 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP3 TP1 EP1] Load weight end. elapsed=156.26 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP2 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP2 TP0 EP0] Load weight end. elapsed=156.30 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:56:40 PP2 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP2 TP3 EP3] Load weight end. elapsed=156.36 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 08:56:41 PP3 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP3 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP3 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP3 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP3 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP3 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP3 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP3 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP2 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP2 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP2 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP2 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP2 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP2 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP2 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP2 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:43 PP3 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:56:43 PP3 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP3 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:56:43 PP3 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:56:43 PP3 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP3 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:56:43 PP3 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP3 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP2 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:56:43 PP3 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP3 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP3 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP3 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP2 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP2 TP0 EP0] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:56:43 PP2 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:56:43 PP2 TP2 EP2] Init torch distributed ends. elapsed=0.01 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP2 TP2 EP2] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:56:43 PP2 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:56:43 PP2 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP2 TP1 EP1] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:56:43 PP2 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:56:43 PP2 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP2 TP3 EP3] Load weight begin. avail mem=36.77 GB +[2026-08-31 08:56:43 PP3 TP1 EP1] Load weight end. elapsed=0.81 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:43 PP3 TP3 EP3] Load weight end. elapsed=0.81 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:43 PP3 TP2 EP2] Load weight end. elapsed=0.81 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:43 PP2 TP3 EP3] Load weight end. elapsed=0.80 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP2 TP1 EP1] Load weight end. elapsed=0.83 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP2 TP2 EP2] Load weight end. elapsed=0.83 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP3 TP0 EP0] Load weight end. elapsed=0.91 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP2 TP0 EP0] Load weight end. elapsed=0.91 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP3 TP0 EP0] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:56:44 PP2 TP0 EP0] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:56:44 PP3 TP2 EP2] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:56:44 PP2 TP2 EP2] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:56:44 PP3 TP3 EP3] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:56:44 PP2 TP1 EP1] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:56:44 PP3 TP1 EP1] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 08:56:44 PP2 TP3 EP3] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 08:56:44 PP3 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:56:44 PP2 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:56:44 PP3 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:44 PP2 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:44 PP3 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP3 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP3 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP3 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP2 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP2 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP2 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP2 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:44 PP2 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:44 PP3 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:44 PP3 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:44 PP3 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:44 PP2 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:44 PP2 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:44 PP2 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:44 PP3 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:44 PP2 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:44 PP2 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP3 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP3 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP2 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP2 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP3 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP3 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP3 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP2 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP2 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP2 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP3 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP2 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP3 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP2 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP2 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP3 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP3 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP3 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP2 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP3 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP3 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP3 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP2 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP2 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP3 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP2 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP2 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP2 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 08:56:45 PP3 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP3 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP3 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP3 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP2 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP2 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP3 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp3_dp0.json +[2026-08-31 08:56:45 PP3 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp3_dp0.json +[2026-08-31 08:56:45 PP3 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp3_dp0.json +[2026-08-31 08:56:45 PP3 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp3_dp0.json +[2026-08-31 08:56:45 PP2 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp2_dp0.json +[2026-08-31 08:56:45 PP2 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp2_dp0.json +[2026-08-31 08:56:45 PP2 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp2_dp0.json +[2026-08-31 08:56:45 PP2 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp2_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:00 [options] + warnings.warn( +[2026-08-31 08:53:18] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:53:18] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:53:18] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:53:21] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:53:21] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 2, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 185891929, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:22] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:22] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:22] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:45 PP4 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:45 PP4 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:49 PP4 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:49 PP4 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:50.020326 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:50.020396 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:50.020407 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:50.020442 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16293 +I0831 08:53:50.020529 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:50.042943 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:50.048530 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:50.054992 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:50.061851 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:50.064541 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:50.064640 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:50.064699 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:50.071813 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:50.073268 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:50.073298 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:50.096875 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:50.098260 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:50.098285 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:50.124886 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:50.126327 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:50.126353 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:50.153002 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:50.154373 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:50.154500 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:50 PP4 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:53:50 PP5 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP5 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:50 PP5 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP5 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:50 PP4 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP4 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:51 PP5 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:51 PP5 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:52 PP4 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:52 PP4 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:53 PP4 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:53 PP4 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:53 PP5 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:53 PP5 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:54 PP4 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:54 PP4 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:54.724299 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:54.724385 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:54.724397 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:54.724432 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16518 +I0831 08:53:54.724572 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:54.730189 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:54.736461 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:54.743657 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:54.751874 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:54.755283 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:54.755370 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:54.755424 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.763515 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:54.764946 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:54.764974 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.773379 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:54.774753 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:54.774799 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.783248 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:54.784755 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:54.784791 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:54.793368 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:54.794791 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:54.794920 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:54 PP4 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:53:55 PP5 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:55 PP5 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:55 PP5 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:55 PP5 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:55.309867 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:55.309934 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:55.309945 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:55.309973 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16196 +I0831 08:53:55.310075 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:55.312630 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:55.312711 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:55.312722 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:55.312752 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16846 +I0831 08:53:55.312870 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:55.335511 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:55.336318 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:55.341210 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:55.343000 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:55.349993 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:55.357013 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:55.359699 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:55.359794 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:55.359850 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.364662 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:55.367072 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:55.368585 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:55.368614 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.371618 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:55.374270 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:55.374361 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:55.374426 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.381544 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:55.382951 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:55.382978 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.392861 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:55.394236 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:55.394261 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.408825 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:55.410187 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:55.410238 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.420902 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:55.422286 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:55.422310 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.429598 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:55.430989 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:55.431115 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:55 PP5 TP0 EP0] Init torch distributed begin. +I0831 08:53:55.436909 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:55.438261 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:55.438289 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.464946 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:55.466281 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:55.466418 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:55 PP5 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:53:55 PP4 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:55 PP4 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:53:55 PP5 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:55 PP5 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:56.215484 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:56.215564 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:56.215574 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:56.215600 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16439 +I0831 08:53:56.215675 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:56.219363 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:56.219435 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:56.219446 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:56.219473 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15761 +I0831 08:53:56.219555 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:56.238736 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:56.243448 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:56.244342 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:56.249984 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:56.250613 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:56.256973 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:56.257606 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:56.260280 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:56.260371 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:56.260427 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.263942 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:56.266618 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:56.266711 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:56.266765 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.273891 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:56.275305 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:56.275334 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.284893 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:56.286381 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:56.286412 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.300920 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:56.302264 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:56.302295 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.312849 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:56.314213 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:56.314239 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.328930 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:56.330289 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:56.330315 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.340758 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:56.342142 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:56.342167 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:56.356981 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:56.358511 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:56.358657 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:56 PP4 TP0 EP0] Init torch distributed begin. +I0831 08:53:56.368831 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:56.370239 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:56.370354 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:56 PP5 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:53:57 PP4 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:57 PP4 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:57.892189 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:57.892266 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:57.892277 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:57.892303 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16175 +I0831 08:53:57.892396 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:57.916260 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:57.923041 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:57.930034 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:57.937009 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:57.939698 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:57.939812 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:57.939867 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:57.947075 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:57.948575 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:57.948670 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:57.972958 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:57.974392 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:57.974427 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.000834 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:58.002223 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:58.002249 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.028934 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:58.030321 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:58.030529 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:58 PP4 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:53:58 PP5 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:58 PP5 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:58.728438 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:58.728498 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:58.728508 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 08:53:58.728530 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16003 +I0831 08:53:58.728608 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:58.751147 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:58.756824 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:58.763230 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:58.770053 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:58.772733 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:58.772823 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:58.772879 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.780045 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:58.781527 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 08:53:58.781561 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.808831 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:58.810292 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 08:53:58.810323 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.836907 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:58.838392 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 08:53:58.838424 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.864945 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:58.866310 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 08:53:58.866464 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:58 PP5 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:54:02 PP4 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:02 PP5 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:03 PP5 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP4 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP5 TP3 EP3] Init torch distributed ends. elapsed=5.03 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP5 TP1 EP1] Init torch distributed ends. elapsed=7.53 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP5 TP2 EP2] Init torch distributed ends. elapsed=8.43 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP5 TP0 EP0] Init torch distributed ends. elapsed=8.47 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP4 TP3 EP3] Init torch distributed ends. elapsed=9.10 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP4 TP2 EP2] Init torch distributed ends. elapsed=13.75 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP4 TP1 EP1] Init torch distributed ends. elapsed=5.87 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP4 TP0 EP0] Init torch distributed ends. elapsed=7.54 s, mem usage=0.58 GB +[2026-08-31 08:54:04 PP4 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP4 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP4 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP4 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP5 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP5 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP5 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP5 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP5 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP5 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP5 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP5 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP4 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP4 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP5 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP4 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP5 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP5 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP4 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP5 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP4 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP5 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP5 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP4 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP5 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP5 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP4 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP4 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP4 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP4 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP4 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP4 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:11 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:13 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:56:40 PP5 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP5 TP1 EP1] Load weight end. elapsed=155.94 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP4 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP4 TP3 EP3] Load weight end. elapsed=155.97 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP5 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP5 TP0 EP0] Load weight end. elapsed=156.02 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP5 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP5 TP3 EP3] Load weight end. elapsed=156.06 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP4 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP4 TP2 EP2] Load weight end. elapsed=156.18 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP4 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP4 TP0 EP0] Load weight end. elapsed=156.09 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP5 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP5 TP2 EP2] Load weight end. elapsed=156.16 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:40 PP4 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:40 PP4 TP1 EP1] Load weight end. elapsed=156.17 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:41 PP5 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP5 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP5 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP5 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP5 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP5 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP5 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP5 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP4 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP4 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP4 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP4 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP4 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP4 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:41 PP4 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:41 PP4 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:43 PP4 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:56:43 PP4 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:56:43 PP4 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:56:43 PP4 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP4 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP4 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP4 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP4 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP4 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP5 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:56:43 PP5 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP5 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP5 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:56:43 PP5 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:56:43 PP5 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:56:43 PP5 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP5 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP5 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP5 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP5 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP5 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP4 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:56:43 PP4 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP4 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:44 PP5 TP3 EP3] Load weight end. elapsed=1.00 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP5 TP2 EP2] Load weight end. elapsed=1.01 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP4 TP2 EP2] Load weight end. elapsed=1.01 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP4 TP3 EP3] Load weight end. elapsed=1.01 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP5 TP0 EP0] Load weight end. elapsed=1.00 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP4 TP0 EP0] Load weight end. elapsed=1.01 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP4 TP1 EP1] Load weight end. elapsed=1.00 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP5 TP1 EP1] Load weight end. elapsed=1.00 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP4 TP2 EP2] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:56:44 PP5 TP1 EP1] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:56:44 PP5 TP2 EP2] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:56:44 PP5 TP3 EP3] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:56:44 PP4 TP1 EP1] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:56:44 PP4 TP3 EP3] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:56:44 PP4 TP0 EP0] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 08:56:44 PP5 TP0 EP0] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 08:56:44 PP4 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:56:44 PP4 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:44 PP5 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:56:44 PP5 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:45 PP5 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP5 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP5 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP4 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP4 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP5 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP4 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP4 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP4 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP5 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP5 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP5 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP4 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP5 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP4 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP4 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP5 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP5 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP5 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP4 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP5 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP4 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP4 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP4 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP5 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP5 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP5 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP5 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP4 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP4 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP4 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP4 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP4 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP5 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP5 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP5 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP4 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP4 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP4 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp4_dp0.json +[2026-08-31 08:56:45 PP4 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp4_dp0.json +[2026-08-31 08:56:45 PP4 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp4_dp0.json +[2026-08-31 08:56:45 PP5 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp5_dp0.json +[2026-08-31 08:56:45 PP4 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp4_dp0.json +[2026-08-31 08:56:45 PP5 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp5_dp0.json +[2026-08-31 08:56:45 PP5 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp5_dp0.json +[2026-08-31 08:56:45 PP5 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp5_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:00 [options] + warnings.warn( +[2026-08-31 08:53:20] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 08:53:20] Attention backend not specified. Use triton backend by default. +[2026-08-31 08:53:20] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 08:53:24] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 08:53:24] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 3, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 954862221, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 08:53:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:50 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:50 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:54 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:54 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:54 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:54 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:55 PP7 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:55 PP7 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:55.454433 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:55.454509 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:55.454519 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:53:55.454551 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15578 +I0831 08:53:55.454643 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:55.477180 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:55.482851 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:55.488567 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:55.495510 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:55.498189 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:55.498296 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:55.498358 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.505467 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:55.506913 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:53:55.506942 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.531476 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:55.532835 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:53:55.532861 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.559511 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:55.560911 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:53:55.560935 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:55.587567 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:55.588934 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:53:55.589079 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:55 PP7 TP3 EP3] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:56 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:56 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:56 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:56 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 08:53:57 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:57 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:57 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:57 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:57 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:53:57 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:53:57 PP6 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:57 PP6 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:58.170172 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:58.170252 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:58.170264 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:53:58.170295 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16883 +I0831 08:53:58.170389 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:58.195024 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:58.204003 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:58.212961 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:58.221884 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:58.225513 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:58.225606 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:58.225665 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.233767 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:58.235211 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:53:58.235301 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.263800 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:58.265254 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:53:58.265296 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.291579 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:58.292982 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:53:58.293018 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:58.319525 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:58.320865 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:53:58.321002 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:58 PP6 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:53:59 PP6 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:53:59 PP6 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:53:59.266732 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:53:59.266809 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:53:59.266822 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:53:59.266858 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16813 +I0831 08:53:59.266958 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:53:59.289808 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:53:59.296327 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:53:59.303045 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:53:59.310088 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:53:59.312767 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:53:59.312862 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:53:59.312920 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:59.320246 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:53:59.321666 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:53:59.321694 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:59.347569 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:53:59.348917 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:53:59.348942 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:59.375478 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:53:59.376849 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:53:59.376874 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:53:59.403505 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:53:59.404847 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:53:59.404963 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:53:59 PP6 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:54:00 PP7 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:54:00 PP7 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:54:00 PP6 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:54:00 PP6 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 08:54:01 PP6 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:54:01.053902 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +[2026-08-31 08:54:01 PP6 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:54:01.053989 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:54:01.054003 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:54:01.054039 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15537 +I0831 08:54:01.054157 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:54:01.065745 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:54:01.065833 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:54:01.065846 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:54:01.065878 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15515 +I0831 08:54:01.065994 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:54:01.078504 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:54:01.085597 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:54:01.090167 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:54:01.092578 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:54:01.097223 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:54:01.099576 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:54:01.102280 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:54:01.102375 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:54:01.102430 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.104256 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:54:01.109555 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:54:01.111070 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:54:01.111099 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.111258 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:54:01.113938 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:54:01.114030 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:54:01.114087 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.121241 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:54:01.122694 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:54:01.122723 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.135416 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:54:01.136818 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:54:01.136844 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.147522 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:54:01.148888 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:54:01.148912 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.163515 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:54:01.164925 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:54:01.164951 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.175585 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:54:01.176961 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:54:01.176994 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.191543 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:54:01.192904 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:54:01.193042 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:54:01 PP7 TP0 EP0] Init torch distributed begin. +I0831 08:54:01.203522 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:54:01.204866 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:54:01.204969 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:54:01 PP6 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:54:01.215193 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:54:01.215260 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:54:01.215269 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:54:01.215289 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16670 +I0831 08:54:01.215353 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:54:01.236409 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:54:01.263381 227 topology.cpp:127] Device mlx5_1 port 1 is available +[2026-08-31 08:54:01 PP7 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:54:01 PP7 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:54:01.270503 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:54:01.295424 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:54:01.298135 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:54:01.298231 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:54:01.298287 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.305483 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:54:01.306931 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:54:01.306963 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.331523 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:54:01.332953 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:54:01.332978 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.359820 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:54:01.361261 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:54:01.361294 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.368708 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:54:01.370076 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:54:01.370199 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:54:01 PP6 TP0 EP0] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:54:01.434119 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:54:01.434170 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:54:01.434177 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:54:01.434198 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15764 +I0831 08:54:01.434274 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +[2026-08-31 08:54:01 PP7 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 08:54:01 PP7 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +I0831 08:54:01.456872 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:54:01.461301 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:54:01.466490 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:54:01.471679 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:54:01.473847 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:54:01.473930 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:54:01.473975 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.479976 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:54:01.481237 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:54:01.481261 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.507539 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:54:01.508924 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:54:01.508955 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.535507 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:54:01.536890 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:54:01.536916 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.564307 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:54:01.565682 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:54:01.565802 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:54:01 PP7 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 08:54:01.619671 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 08:54:01.619746 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 08:54:01.619753 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 08:54:01.619773 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16681 +I0831 08:54:01.619841 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 08:54:01.640211 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 08:54:01.644717 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 08:54:01.649175 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 08:54:01.652968 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 08:54:01.654039 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 08:54:01.654084 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 08:54:01.654107 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.657271 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 08:54:01.658015 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 08:54:01.658025 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.661531 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 08:54:01.662983 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 08:54:01.663021 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.687623 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 08:54:01.689018 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 08:54:01.689044 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 08:54:01.696393 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 08:54:01.697729 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 08:54:01.697866 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 08:54:01 PP7 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:54:02 PP6 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:02 PP7 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 08:54:02 PP7 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:02 PP7 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP6 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 08:54:03 PP7 TP2 EP2] Init torch distributed ends. elapsed=2.31 s, mem usage=0.50 GB +[2026-08-31 08:54:03 PP7 TP1 EP1] Init torch distributed ends. elapsed=2.18 s, mem usage=0.50 GB +[2026-08-31 08:54:03 PP7 TP0 EP0] Init torch distributed ends. elapsed=2.68 s, mem usage=0.50 GB +[2026-08-31 08:54:03 PP6 TP2 EP2] Init torch distributed ends. elapsed=2.67 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP6 TP3 EP3] Init torch distributed ends. elapsed=5.56 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP6 TP0 EP0] Init torch distributed ends. elapsed=2.51 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP6 TP1 EP1] Init torch distributed ends. elapsed=4.47 s, mem usage=0.58 GB +[2026-08-31 08:54:03 PP7 TP3 EP3] Init torch distributed ends. elapsed=8.29 s, mem usage=0.50 GB +[2026-08-31 08:54:04 PP7 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP6 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP7 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP6 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP7 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP6 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP7 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP6 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP6 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP6 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP7 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP6 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 08:54:04 PP7 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP6 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP6 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP7 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP7 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-08-31 08:54:04 PP7 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP6 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP7 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 08:54:04 PP6 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP7 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP6 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP6 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP7 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP6 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP6 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP7 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP6 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP7 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 08:54:04 PP7 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:54:04 PP7 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:09 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:55:21 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 08:56:33 PP6 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP6 TP1 EP1] Load weight end. elapsed=149.13 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:33 PP7 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP7 TP3 EP3] Load weight end. elapsed=149.22 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:56:33 PP7 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP7 TP2 EP2] Load weight end. elapsed=149.24 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:56:33 PP6 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP6 TP2 EP2] Load weight end. elapsed=149.29 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:33 PP7 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP7 TP0 EP0] Load weight end. elapsed=149.49 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:56:33 PP7 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP7 TP1 EP1] Load weight end. elapsed=149.39 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 08:56:33 PP6 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:33 PP6 TP3 EP3] Load weight end. elapsed=149.54 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:34 PP6 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 08:56:34 PP6 TP0 EP0] Load weight end. elapsed=149.51 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 08:56:34 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:34 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 08:56:34 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 08:56:43 PP7 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:56:43 PP7 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP7 TP2 EP2] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:56:43 PP7 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:56:43 PP7 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP7 TP1 EP1] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:56:43 PP6 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:56:43 PP6 TP1 EP1] Init torch distributed begin. +[2026-08-31 08:56:43 PP6 TP2 EP2] Init torch distributed begin. +[2026-08-31 08:56:43 PP6 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP6 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP6 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP6 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP6 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP6 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP7 TP0 EP0] Init torch distributed begin. +[2026-08-31 08:56:43 PP7 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP6 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:56:43 PP7 TP0 EP0] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:56:43 PP6 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP6 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 08:56:43 PP7 TP3 EP3] Init torch distributed begin. +[2026-08-31 08:56:43 PP7 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 08:56:43 PP7 TP3 EP3] Load weight begin. avail mem=32.33 GB +[2026-08-31 08:56:44 PP7 TP3 EP3] Load weight end. elapsed=0.79 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:56:44 PP6 TP1 EP1] Load weight end. elapsed=0.80 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP7 TP2 EP2] Load weight end. elapsed=0.83 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:56:44 PP7 TP1 EP1] Load weight end. elapsed=0.83 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:56:44 PP6 TP2 EP2] Load weight end. elapsed=0.83 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP6 TP3 EP3] Load weight end. elapsed=0.83 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP7 TP0 EP0] Load weight end. elapsed=0.90 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 08:56:44 PP6 TP0 EP0] Load weight end. elapsed=0.91 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 08:56:44 PP6 TP1 EP1] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:56:44 PP7 TP2 EP2] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:56:44 PP7 TP1 EP1] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:56:44 PP6 TP0 EP0] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:56:44 PP6 TP2 EP2] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:56:44 PP7 TP0 EP0] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:56:44 PP6 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:56:44 PP7 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 08:56:44 PP6 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:44 PP7 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 08:56:44 PP6 TP3 EP3] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 08:56:44 PP7 TP3 EP3] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 08:56:45 PP7 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP7 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP7 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP7 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP6 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP6 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP6 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP6 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 08:56:45 PP6 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP6 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP7 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP7 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP6 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP7 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP7 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 08:56:45 PP6 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 08:56:45 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP6 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP6 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP6 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:56:45 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:56:45 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:56:45 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 08:56:45 PP7 TP3 EP3] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:56:45 PP7 TP2 EP2] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:56:45 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 08:56:45 PP6 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP7 TP0 EP0] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:56:45 PP7 TP1 EP1] Memory pool end. avail mem=22.26 GB +[2026-08-31 08:56:45 PP6 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP6 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP6 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 08:56:45 PP6 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 08:56:45 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:56:45 PP7 TP2 EP2] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:56:45 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:56:45 PP7 TP1 EP1] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:56:45 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:56:45 PP7 TP3 EP3] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:56:45 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 08:56:45 PP7 TP0 EP0] Memory pool end. avail mem=11.88 GB +[2026-08-31 08:56:45 PP6 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP6 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP7 TP2 EP2] Overriding draft attention backend to triton. +[2026-08-31 08:56:45 PP7 TP1 EP1] Overriding draft attention backend to triton. +[2026-08-31 08:56:45 PP7 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 08:56:45 PP7 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 08:56:45 PP7 TP0 EP0] Overriding draft attention backend to triton. +[2026-08-31 08:56:45 PP7 TP3 EP3] Overriding draft attention backend to triton. +[2026-08-31 08:56:45 PP6 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp6_dp0.json +[2026-08-31 08:56:45 PP6 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp6_dp0.json +[2026-08-31 08:56:45 PP6 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp6_dp0.json +[2026-08-31 08:56:45 PP6 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp6_dp0.json +[2026-08-31 08:56:45 PP7 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp7_dp0.json +[2026-08-31 08:56:45 PP7 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp7_dp0.json +[2026-08-31 08:56:45 PP7 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp7_dp0.json +[2026-08-31 08:56:45 PP7 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp7_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:00