From 3fbfb84e12a65f02361c5503820e7b14b715197f Mon Sep 17 00:00:00 2001 From: Zhiyi Hong <2497491955@qq.com> Date: Mon, 31 Aug 2026 18:29:06 +0800 Subject: [PATCH] fix: validate Kimi PP8 DFlash PD on GSM8K C1 and C8 --- README.md | 2 + docs/Kimi-K3_PP与DFlash迁移审计.md | 20 +- ...mi_k3_pp_dflash_pr33863_transfer_plan.json | 32 + .../README.md | 191 +- .../deploy_pd_dflash.sh | 4 +- .../results/cpu_regression_20260831_final.log | 31 + .../final_image_gpu_container_20260831.json | 427 ++++ .../final_image_gpu_verify_20260831.log | 18 + .../bench/gsm8k_c1.dataset_path_failure.log | 16 + .../bench/gsm8k_c1.jsonl | 64 + .../bench/gsm8k_c1.log | 28 + .../bench/gsm8k_c1_summary.json | 22 + .../bench/gsm8k_c8.jsonl | 64 + .../bench/gsm8k_c8.log | 28 + .../bench/gsm8k_c8_summary.json | 22 + .../bench/smoke.json | 1 + .../bench_resume_exit_code | 1 + .../cleanup_exit_code | 1 + .../commands/d_0.cmd.txt | 1 + .../commands/d_1.cmd.txt | 1 + .../commands/d_2.cmd.txt | 1 + .../commands/d_3.cmd.txt | 1 + .../commands/gsm8k_c1.cmd.txt | 1 + .../gsm8k_c1.dataset_path_failure.cmd.txt | 1 + .../commands/gsm8k_c8.cmd.txt | 1 + .../commands/p_0.cmd.txt | 1 + .../commands/p_1.cmd.txt | 1 + .../commands/p_2.cmd.txt | 1 + .../commands/p_3.cmd.txt | 1 + .../commands/router.cmd.txt | 1 + .../exit_code | 1 + .../logs/d_0.log | 1884 +++++++++++++++++ .../logs/d_1.log | 1641 ++++++++++++++ .../logs/d_2.log | 1629 ++++++++++++++ .../logs/d_3.log | 1617 ++++++++++++++ .../logs/p_0.log | 1126 ++++++++++ .../logs/p_1.log | 962 +++++++++ .../logs/p_2.log | 963 +++++++++ .../logs/p_3.log | 972 +++++++++ .../logs/router.log | 144 ++ .../metadata/dataset_mount_validation.txt | 1 + .../metadata/gpu_during_gsm8k.txt | 80 + .../metadata/post_run_resources.txt | 80 + .../metadata/preflight.txt | 8 + .../result_validation.json | 246 +++ 45 files changed, 12249 insertions(+), 89 deletions(-) create mode 100644 docs/evidence/kimi_k3_pp_dflash_pr33863_transfer_plan.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/cpu_regression_20260831_final.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_container_20260831.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_verify_20260831.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.dataset_path_failure.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1_summary.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c8.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c8.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c8_summary.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/smoke.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench_resume_exit_code create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/cleanup_exit_code create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/d_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/d_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/d_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/d_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/gsm8k_c1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/gsm8k_c1.dataset_path_failure.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/gsm8k_c8.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/p_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/p_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/p_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/p_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/commands/router.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/exit_code create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_2.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_1.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_2.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/router.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/dataset_mount_validation.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/gpu_during_gsm8k.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/post_run_resources.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/preflight.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/result_validation.json diff --git a/README.md b/README.md index 45d070f..5832896 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # sskj — 多平台大模型推理性能基准测试项目 +**更新（2026-08-31 18:22:43 CST）**：完成 Kimi-K3 PP8 + DFlash 八节点 PD 部署验证。P 601-604 为 TP4/PP8/EP4，D 605-608 为 TP32/PP1/EP4，均使用 FlashInfer MXFP4、BF16 KV、8K Chunk。固定 GSM8K 64 题 C1/C8 共 128/128 请求成功，答案检查全部通过且无输出截断；按 verify 步数加权的接受长度为 6.4211/6.5158，整体 Output TPS 为 33.38/117.56。修复评测容器的数据集路径，保留原失败与续跑证据；结果、完整服务日志及资源清理记录已归档。历史 5-shot 与前 5 题重叠，本轮作为部署回归而非独立准确率评测。详见 [实现与结果](experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md)。 + **更新（2026-08-31 17:50:51 CST）**：修复 Kimi 混合注意力缓存池在 PD 初始化时缺少 `end_layer` 的兼容问题，按完整注意力层偏移和本地层数确定传输范围，保留普通池与 layer-shard 路径。新增回归后 24 项 CPU 测试通过，八节点修复镜像摘要一致；P 组已完成预热和 Mooncake 注册，健康检查返回 200，D 组正在验证启动。当前 Run 为 `pd-dflash-kvbounds1-20260831-1740`，尚无 GSM8K 验收结果。详见 `experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md`。 **更新（2026-08-31 17:05:00 CST）**：新增 Kimi-K3 PP8 + DFlash PD 适配验证入口。基于 SGLang PR #33863 固定源码，接通 PP 分段 hidden 投影、P 侧 prompt draft KV 生成、D 侧输入生命周期与 TP4→TP32 的 draft GQA KV 传输，保留 Kimi SM120 FlashInfer MXFP4 接入。修复 P 普通预热误带 DFlash verify metadata 的启动问题，23 项 CPU 回归通过；修复镜像已同步 601–608，服务级验证正在进行，尚无 GSM8K 结果。配置为 P TP4/PP8/EP4、D TP32/PP1/EP4、BF16 KV、8K Chunk，计划固定 64 题 C1/C8。详见 `experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md`。 diff --git a/docs/Kimi-K3_PP与DFlash迁移审计.md b/docs/Kimi-K3_PP与DFlash迁移审计.md index 0d1fd5e..0cdf05c 100644 --- a/docs/Kimi-K3_PP与DFlash迁移审计.md +++ b/docs/Kimi-K3_PP与DFlash迁移审计.md @@ -1,18 +1,18 @@ # Kimi-K3：从 PP + DSpark 迁移到 PP + DFlash -更新：2026-08-31；PP、传输与 MoE 集成首版。 +更新：2026-08-31；PP8 + PD + DFlash 集成与 GSM8K 64 题 C1/C8 验证完成。 ## 1. 决策与范围 **优先推进 DFlash 的 PP + PD 适配。** #33863 的分段投影设计能复用于 DFlash，但需要补齐 DFlash worker、调度入口和异构 TP 下的 draft KV 传输。 -当前已实现 DFlash PP worker、Kimi 跨 stage capture、PD 输入衔接、异构 TP 的 draft KV 传输，以及 Kimi FlashInfer 布局/SiTU 接入。P 端两次启动分别暴露普通预热误带 verify metadata、混合池缺少 `end_layer` 的问题，均已针对实际调用点修复，24 项 CPU 测试通过。SM120 编译及 Kimi SiTU GPU 集成回归已通过。Run `pd-dflash-kvbounds1-20260831-1740` 的 P 组健康检查已返回 200，D 组正在启动；尚无跨组请求和 GSM8K 结果。详见 [实现进度与命令](evidence/kimi_k3_pd_dflash/README.md)。 +已实现 DFlash PP worker、Kimi 跨 stage capture、PD 输入衔接、异构 TP 的 draft KV 传输及 Kimi FlashInfer 布局/SiTU 接入。24 项 CPU 回归及 SM120 GPU 集成回归通过。Run `pd-dflash-kvbounds1-20260831-1740` 已完成跨组 smoke 与 GSM8K 64 题 C1/C8：两轮各 64/64 请求和答案检查通过，均无输出截断；按验证步数加权的接受长度为 6.4211/6.5158，整体 Output TPS 为 33.38/117.56。八节点实验容器已清理、GPU 已释放。详见 [实现、结果与实际命令](../experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md)。 暂停 EAGLE3 baseline。KV cache 保持 BF16，chunk 保持 8192。FlashInfer 使用官方 0.6.18 加已合并 #4460 的显式 backport；原版 0.6.18 wheel 尚未包含所需 CUTLASS SiTU 接口。 -目标拓扑：P 使用 601–604 的 TP4/PP8/EP4 与已适配的 FlashInfer MXFP4；D 使用 605–608 的 TP32/PP1，接入 DFlash。这里适配的是 **P 侧 PP 生成 draft 上下文**，D 侧投机执行仍为 PP1。 +已验证拓扑：P 使用 601–604 的 TP4/PP8/EP4；D 使用 605–608 的 TP32/PP1/EP4，两组均为 FlashInfer MXFP4。这里适配的是 **P 侧 PP 生成 draft 上下文**，D 侧投机执行仍为 PP1。 -审计固定版本： +前期源码审计快照（PR 状态为该次审计记录，不表示实时状态）： | 对象 | 版本 | |---|---| @@ -91,14 +91,14 @@ PR 当前 [Mooncake flat 传输分支](https://github.com/sgl-project/sglang/blo 具体用例：D buffer 起始地址设为 2000000，目标 page ID=2。正确目标地址为 `2000000 + 2×16384 = 2032768`，当前函数生成 `2065536`，复制 32768 bytes 而不是单 head 页的 16384 bytes。 -这是特定布局的地址规划复现，尚未启动完整服务复现。迁移时必须加入每条目的布局校验，并实现 GQA head-aware 传输；不能直接复用 flat copy。还需确认 D 接收到全部 target/draft 组件后才进入首轮 draft。 +这是原 PR 的特定布局地址规划复现，未在 GPU 上故意执行越界复制。当前实现已补齐每条目布局校验及 GQA head-aware 传输，CPU 测试覆盖全部 P-TP4→D-TP32 head 映射；修复版本的完整服务已通过跨组请求和 C1/C8 评测。原 flat-copy 失败证据保留用于说明修复动机。 -## 5. 实现与验收顺序 +## 5. 验收与结果口径 -1. **局部数学与协议验证。** 检查各 stage capture 层覆盖、feature 顺序、空 capture stage、单次 RMSNorm；比较拼接投影与分段投影误差。测试 TP4→TP32 的全部 8 个 draft KV head、K/V、6 层、页索引以及失败清理。源/目标布局不兼容时在传输前报错。 -2. **最小 PD 请求。** 保留 P TP4/PP8、D TP32/PP1，先 C1。核对真实 buffer metadata、draft KV 到达情况、首轮 draft logits，以及成功/失败后的资源释放。 -3. **GSM8K 固定 64 题。** 相同 prompt/template、采样设置，依次测试 C1/C8；保留准确率、完成率、接受长度、接受直方图、Output TPS、总耗时与显存状态。当前客户端非流式，不把总耗时换算成 TTFT/ITL。输入不裁剪，输出上限 512，报告截断，不新增 16K synthetic 测试。历史 5-shot 与前 64 题有重叠，本轮沿用以便部署回归，不能作为独立无泄漏的模型准确率评估。 -4. **验收后决定继续投入。** DFlash 正确性与部署通过且有实际 Decode 收益，继续 PD 调优；若关键适配无法通过，或接受行为同样异常，切换到下面的 D-only 诊断。 +1. **局部数学与协议验证已通过。** 覆盖各 stage capture、feature 顺序、空 capture stage、单次 RMSNorm、BF16/FP32 分段投影与 TP4→TP32 传输规划。CPU 方法级回归不等于完整模型 logits 等价验证。 +2. **最小 PD 请求已通过。** P TP4/PP8、D TP32/PP1，经 Router 返回 128 token，服务日志有实际 RDMA ready ACK，D Decode 使用 CUDA Graph。 +3. **GSM8K 固定 64 题 C1/C8 已完成。** 两轮各 64 个响应均成功，独立解析最终答案也全部通过；总耗时 195.42/55.55 秒，无 512-token 截断。完整逐题输出、直方图、服务日志和资源清理证据见结果目录。客户端非流式，不把总耗时换算成 TTFT/ITL。 +4. **统计口径已复核。** 接口的每题等权接受长度为 6.9433/7.0070；直方图按 verify 步数加权为 6.4211/6.5158。两者权重和首尾 token 处理不同，不混用。5-shot 与前 5 题重叠，排除后剩余 59 题也均通过；这仍是历史口径部署回归，不是独立无泄漏的模型能力评测。本轮不重跑 no-spec，因此不宣称投机加速倍数。 真实实验代码继续维护在 601 的 `/data/hzy/sskj` 工作区，不改同事部署目录，不再创建分支。本轮核实当前检出名为 `hzy-kimi3-pd-pp8-standard`，保持现状；独立 SGLang 源码位于 `/data/hzy/src/sglang-kimi-pp-dflash-33863`。 diff --git a/docs/evidence/kimi_k3_pp_dflash_pr33863_transfer_plan.json b/docs/evidence/kimi_k3_pp_dflash_pr33863_transfer_plan.json new file mode 100644 index 0000000..1c66870 --- /dev/null +++ b/docs/evidence/kimi_k3_pp_dflash_pr33863_transfer_plan.json @@ -0,0 +1,32 @@ +{ + "upstream_commit": "6465a6f3d3b6c8b7fee40fba0fdc09cf5e9ca1c5", + "scope": "Extracted upstream functions, synthetic pointer metadata, no GPU or network I/O", + "source_sha256": { + "transfer": "2f5a73b256e040966e181a5695fb4fa97e9fc7fb5c4760832e180b0683820cea", + "pairing": "47bc97bfb649d6564c870113d276a1043230271f2d8924e8d41744cea12c971a" + }, + "results": [ + { + "case": "equal_tp_control", + "source_page_stride_bytes": 16384, + "destination_page_stride_bytes": 16384, + "actual_source_address": 1016384, + "actual_destination_address": 2032768, + "expected_destination_address": 2032768, + "actual_copy_bytes": 16384, + "one_destination_head_page_bytes": 16384, + "layout_matches": true + }, + { + "case": "P_TP4_to_D_TP32_DFlash", + "source_page_stride_bytes": 32768, + "destination_page_stride_bytes": 16384, + "actual_source_address": 1032768, + "actual_destination_address": 2065536, + "expected_destination_address": 2032768, + "actual_copy_bytes": 32768, + "one_destination_head_page_bytes": 16384, + "layout_matches": false + } + ] +} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md index eedc8e4..5875376 100644 --- a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/README.md @@ -1,109 +1,146 @@ -# Kimi-K3 PP + DFlash 适配进度 +# Kimi-K3 PP8 + DFlash PD：实现与验证 -2026-08-31：PP worker、PD 输入衔接、异构 TP 传输和 Kimi FlashInfer 接入已完成首版；SM120 编译及 Kimi SiTU GPU 集成回归通过。两项启动兼容修复及 24 项 CPU 回归已完成。Run `pd-dflash-kvbounds1-20260831-1740` 的 P 组已通过预热、Mooncake 初始化和健康检查（HTTP 200），D 组正在启动；尚无跨组请求或 GSM8K 结果。 +更新：2026-08-31。**八节点部署与固定 GSM8K 64 题 C1/C8 验证完成，128/128 请求成功，答案检查均通过，无输出截断。** 本次完成的是用户后续指定的 DFlash 路线，未恢复 EAGLE3/no-spec 实验。 -## 源码和环境 +## 1. 目标与实现 -- 基于官方 PR #33863 的 `6465a6f3d3b6c8b7fee40fba0fdc09cf5e9ca1c5`。 -- 上游 Python 目录的 4619 个文件均与 Git blob SHA1 校验一致。 -- `pp_dflash_integration.patch` 包含完整 SGLang 增量；`pp_dflash_files.json` 记录文件 SHA256。`mixed_kv_transfer.patch` 保留传输层首版记录。 -- 601 独立源码路径：`/data/hzy/src/sglang-kimi-pp-dflash-33863`。 -- 601 实验路径：`/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation`。 -- 不修改原 EAGLE3/DSpark 部署；不创建新分支。 +保留 P 侧 PP8 的部署方案，让 P 生成真实 DFlash prompt KV，经 Mooncake 传给 D，D 接着执行投机解码。 -## 本轮实现 +```text +P 601-604：TP4 / PP8 / EP4 + 各 stage 采集 target hidden，计算局部投影 + → PP stage 间累加投影结果 + → 最后 stage 统一 RMSNorm，生成 6 层 draft KV + → 传输 target MLA KV、KDA 状态及 draft KV +D 605-608：TP32 / PP1 / EP4 + 接收上下文 → DFlash 提议 → target 验证 → 提交 token +``` -| 部分 | 实现 | +基于 SGLang PR #33863 固定提交 `6465a6f3d3b6c8b7fee40fba0fdc09cf5e9ca1c5`。分段投影复用该 PR 的设计；新增 DFlash worker、Kimi capture、PD 调度衔接和异构 TP 传输，保留已验证的 Kimi SM120 FlashInfer MXFP4 接入。 + +| 模块（相对于 `python/sglang/srt/`） | 本次改动 | |---|---| -| Kimi hidden 采集 | 接入 DFlash checkpoint 的 capture 层；PP 边界由下一 stage 使用实际 residual 聚合权重采集 | -| 分段投影 | 各 stage 计算对应权重列的局部投影，通过 PP proxy 累加；末 stage 统一 RMSNorm 并写 6 层 draft KV | -| P 侧 draft 执行 | 非末 stage 只保留最小 KV pool；P 不执行 draft decode CUDA Graph，D 的原有初始化路径保留 | -| Scheduler 与 PD | 接通 DFlash proxy、next draft input、FutureMap 发布与 idle 生命周期；仅放开 P 侧 PP | -| Mooncake 传输 | 注册每 entry 的 dtype/head/page stride/容量；target MLA 整页传输，draft GQA 按 head 交集逐 token 切片；写入前完成全部边界检查 | -| FlashInfer | 迁移已验证的 Kimi SM120 布局与 SiTU 参数适配，同时保留新上游 SwigluStep 行为 | +| `models/kimi_k3.py`、`speculative/dflash_pp.py` | 按 checkpoint 采集指定层；处理 PP 边界 residual；分段投影后只做一次归一化 | +| `speculative/dflash_worker_v2.py` | 接收、传递 PP 上下文；最后 stage 写 draft KV；非末 stage 保留最小 KV pool | +| `managers/scheduler_pp_mixin.py`、`speculative/spec_info.py` | 接通 DFlash proxy、PD 输入、FutureMap 与 idle 生命周期 | +| `disaggregation/common/kv_entry_layout.py`、`mooncake/conn.py` 等 | target MLA 整页传输；draft GQA 按 head 交集逐 token 切片；独立计算源/目标 stride，在提交前检查边界 | +| `model_executor/runner/base_runner.py`、`disaggregation/prefill.py` | 修复 P 普通预热误带 verify metadata，以及混合 KV 池传输层范围计算 | +| `arg_groups/` | 仅放开已实现的 P 侧 PP + DFlash，D 侧保持 PP1 | -目标范围：Kimi hybrid MLA + 普通 NHD DFlash draft KV、Mooncake、CP=DCP=1，无 staging/unified KV/HiSparse。P TP4/PP8/EP4；D TP32/PP1；BF16 KV；chunk=8192。 +完整增量见 [pp_dflash_integration.patch](pp_dflash_integration.patch)，19 个文件校验值见 [pp_dflash_files.json](pp_dflash_files.json)。独立源码位于 601 的 `/data/hzy/src/sglang-kimi-pp-dflash-33863`，不覆盖同事部署源码。 -## FlashInfer 依赖 +## 2. 实际配置 -官方 `#4460` 已于 2026-08-21 合并，提交为 `b460bc00cb373541102d2155aec35bd626e522ce`。但 `v0.6.18` 属于另一条发布分支，实际下载的官方 wheel 没有 CUTLASS `situ_beta/situ_linear_beta` API。 +| 项目 | 本次值 | +|---|---| +| GPU | 8 节点 × 8 张 RTX 6000D；每卡 85651 MiB，SM120，PCIe | +| P / D 并行 | TP4/PP8/EP4；TP32/PP1/EP4 | +| MoE / A2A | `flashinfer_mxfp4` / `none`，两组一致 | +| Target / Draft KV | 均为 `bfloat16` | +| Chunk / Page | 8192 / 64 | +| 最大活跃请求 / Decode Graph BS | 8 / 8；D 日志确认实际 Decode 使用 CUDA Graph | +| P / D 显存比例 | 0.88 / 0.86 | +| P / D Mamba memory ratio | 0.36 / 0.21 | +| Prefix Cache | `--disable-radix-cache`；128 个响应的 cached tokens 均为 0 | +| RDMA | Mooncake，`mlx5_0,mlx5_1,mlx5_2,mlx5_3`，容器挂载 `/dev/infiniband` | +| Draft | `/data/hf_models/Kimi-K3-DFlash`，6 层，hidden 7168，32 Q heads / 8 KV heads，head dim 128 | +| Draft 上下文 | Sliding window 4096，RoPE theta 2000000，capture `[19,37,54,66,78,90]` | +| Draft block | `--speculative-num-draft-tokens 16`；每个 verify step 统计 15 个 draft 候选位置 | -因此本镜像使用 **官方 0.6.18 wheel + 未改写的 #4460 合并补丁**。Dockerfile 对 2 个 Python 和 5 个 C++ 文件先执行 `git apply --check`，再应用上游补丁。它不是未经修改的官方 0.6.18，也不是旧的私人 SiTU kernel。 +当前镜像：`local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1`。Linux/amd64 manifest 为 `e74c9b1e7fb6a9509db32576d2c9196b42ef329118909eeebd620cb9db2f91e9`；image config 为 `26584138eb1e097183ce167cbffe4198be0611e0cfde7422cbe692100d865982`。八节点镜像身份已核对。 -移除镜像内旧版 `flashinfer-cubin`、`flashinfer-jit-cache`，保留版本校验；SM120 CUTLASS 通过官方 JIT 路径编译。[安装说明](https://docs.flashinfer.ai/installation.html)、[#4460](https://github.com/flashinfer-ai/flashinfer/pull/4460)。 +FlashInfer 使用**官方 0.6.18 wheel + 已合并 #4460 的原样 backport**（`b460bc00cb373541102d2155aec35bd626e522ce`）。该 wheel 所属发布分支尚无 CUTLASS SiTU API，不能描述为“未经修改的 0.6.18 直接支持”。移除旧 cubin/JIT-cache 包，保留版本检查；编译缓存挂载到 `/data/hzy/cache/kimi-dflash-fi0618-situ4460`。最后两项启动修复仅替换 Python 层，不改变 FlashInfer kernel。 -## CPU 验证 +[Dockerfile](Dockerfile) 的完整重建输入保留在 601 本实验 `results/`：`sglang-kimi-pp-dflash-33863-integrated.tar.gz`（SHA256 `226065165bbb5f85ce516b847ff6da7e5aa823f12e69b87558e67c91996e3ba0`）和 `official_flashinfer_0618/flashinfer_python-0.6.18-py3-none-any.whl`（SHA256 `d5d26edb48f8def0bf28b2c94415aef7f0754e2cd9e3cb3d6e7081262d068aad`）。源码归档含固定上游 Python 树、补丁后的代码、3 个测试文件及 `SOURCE_MANIFEST.json`；GPU 测试和服务验证状态以外部结果日志为准，不回写构建时的历史 manifest。 -```bash -# 独立传输测试，无需 GPU 或 torch -cd /data/hzy/src/sglang-kimi-pp-dflash-33863 -python3 test/registered/disaggregation/test_mixed_kv_entry_layout.py +## 3. 测试结果 -# 全部 CPU 回归需要带 torch 的容器；镜像构建时自动执行 -python3 -m unittest discover -s test/registered/disaggregation -v +Run ID：`pd-dflash-kvbounds1-20260831-1740`。使用 GSM8K 实际文件前 64 题、5-shot、temperature=0、top_p=1，输入不裁剪，输出上限 512。实际输入 694-794 token，最长输出 225 token。C1 先运行，随后 C8；没有新增 no-spec 或 16K synthetic 测试。 + +| 指标 | C1 | C8 | +|---|---:|---:| +| 请求成功 / 计划 | 64/64 | 64/64 | +| 答案检查通过 | 64/64 | 64/64 | +| 排除与 5-shot 重叠的前 5 题后 | 59/59 | 59/59 | +| 达到输出上限 | 0 | 0 | +| 客户端总耗时 | 195.42 s | 55.55 s | +| 总输出 token | 6523 | 6530 | +| 整体 Output TPS | 33.38 | 117.56 | +| 接口接受长度，每题等权平均 | 6.9433 | 7.0070 | +| 直方图接受长度，按 verify 步数加权 | **6.4211** | **6.5158** | +| Draft 接受率，不含 bonus | 36.14% | 36.77% | +| Verify 步数 | 1026 | 1012 | +| 接受 / 提议 draft token | 5562 / 15390 | 5582 / 15180 | +| 请求回撤 | 0 | 0 | + +**这套 PP8 + PD + DFlash 已能完成本轮部署回归，接受长度没有停留在接近 1 的状态。** C8 的 117.56 token/s 是整体输出吞吐，包含 Prefill、传输、调度及 Decode，不是单请求 Decode 速度。本轮未重跑匹配的 no-spec 对照，不据此宣称投机加速倍数。 + +历史客户端的 5-shot 取自同一测试文件前 5 题，与这 64 题重叠，保留此口径用于历史部署回归。剩余 59 题也全部通过；本报告不将此结果作为独立无泄漏的模型能力评测。所有输出均包含 `####` 最终答案，已独立按带符号数值重新解析并与标签比对。 + +### 接受长度为什么有两种数值 + +接口 `spec_accept_length` 在 `tokenizer_manager.py:2814` 定义为 `completion_tokens / spec_verify_ct`，受 P 侧首 token 和最后一步停止截断影响。例如一题输出 49 token、验证 3 次，接口值是 16.33；实际直方图为 3 次各接受 15 个 draft，含 bonus 的每步长度为 16，没有超出 block 能力。 + +```text +加权接受长度 = 1 + 总接受 draft token / 总 verify 次数 +C1 = 1 + 5562 / 1026 = 6.4211 +C8 = 1 + 5582 / 1012 = 6.5158 ``` -| 回归 | 数量 | 主要覆盖 | -|---|---:|---| -| 传输 | 7 | 144 组 KV-head/TP 组合、实际 P-TP4→D-TP32 全 32 rank、逐字节复制、越界/重复写拒绝、注册往返 | -| PP 与 PD 输入 | 13 | PP1/2/4/8/16 capture 归属、边界 residual、空 capture stage、BF16/FP32 分段投影、单次 norm、末 stage 写 KV、输入生命周期、P 普通预热不创建 verify metadata、混合池/普通池传输层范围 | -| Kimi MoE 合并 | 4 | 两类 gate/up 布局、SiTU 参数、activation 白名单、非连续输入、保留 SwigluStep、API 能力检查 | +128 题逐一验证了 `sum(histogram)=verify_ct`、直方图加权计数等于 `accepted_drafts`、`proposed_drafts=15×verify_ct`。接口原值完整保留。客户端是非流式，不由总耗时推造 TTFT/ITL。 -24 项在本地与 601 `kvbounds1` 镜像中通过。此前 `pwarm1` 镜像的 23 项也在 602 导入容器中通过，并完成真实 DFlashWorkerV2、MooncakeKVManager 导入。8 条启动命令的 CLI 解析及两类服务的参数后处理已检查。CPU 测试对实际方法作 AST 提取，使用 CPU tensor 或记录型 engine，核验数学与调用契约，不替代服务级验证。 +## 4. 验证与运行状态 -601 GPU6 的 `test_kimi_k3_sm120_situ_layout_and_noncontiguous_input` 已通过：检查 Kimi gate/up 及 scale 布局、SiTU(4,25)、非连续输入，以及 SGLang adapter 与直接 FlashInfer 调用的输出一致性。这是小 shape 的集成回归；端到端正确性由后续 PD/GSM8K 检验。 +- 24 项 CPU 回归通过：13 项 PP/PD、7 项传输、4 项 Kimi MoE，覆盖 PP1/2/4/8/16、BF16/FP32 分段投影、TP4→TP32 全部 GQA head 映射、越界拒绝及输入生命周期。 +- 601 GPU6 的 Kimi SM120 SiTU 布局与非连续输入集成回归通过；固化镜像独立 GPU 复测 `1 passed`。CPU 回归使用实际方法的 AST 提取与 CPU tensor，不是完整模型 logits 等价测试。 +- Router smoke 和正式 C1/C8 全部完成。D 日志有 RDMA ready ACK 与实际 Decode CUDA Graph 记录。 +- 本 Run 服务日志未检出 Traceback、CUDA OOM、KVTransferError 或非法显存访问。存在 NUMA 查询权限警告和 K3 fused KDA decode 布局回退提示，未阻断本轮；后续性能调优可单独审查。 +- 运行中显存快照：P 每卡 53361-76579 MiB，D 每卡 79849-80019 MiB。这是离散快照，不是全程峰值。 +- 续跑和清理均退出 0。601-608 本实验容器已删除，GPU compute process 为空，显存回落到 0-3 MiB。 -修复前镜像 `local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460` 的独立 GPU 复测通过，Mooncake CUDA engine 导入成功，GPU 测试 1 passed、25.28 秒（复用 JIT 缓存）。 +## 5. 如何运行 -当前镜像为 `local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1`，在 `pwarm1` 上只替换 Python 层范围处理及回归测试，不改变 FlashInfer 层或 GPU kernel。Linux/amd64 manifest 为 `sha256:e74c9b1e7fb6a9509db32576d2c9196b42ef329118909eeebd620cb9db2f91e9`，image config 为 `sha256:26584138eb1e097183ce167cbffe4198be0611e0cfde7422cbe692100d865982`；OCI index 为 `sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f`。区分这三个摘要，不将 Docker 不同模式显示的 ID 当成代码不一致。 - -601 证据均位于实验目录 `results/`： - -- `pp_and_transport_cpu_601_20260831.log`：18 项 CPU 回归。 -- `patched_worker_import_20260831.log`、`patched_launch_help_20260831.log`：真实模块导入与 CLI。 -- `image_build_situ4460_cpucheck_20260831.log`：集成镜像的完整 API/回归检查。 -- `command_parse_20260831.log`：P/D 全部 8 条命令通过 `ServerArgs` argparse 检查；该检查不执行 ServerArgs 后处理或服务初始化。 -- `args_resolve_20260831.log`、`resolved_p_20260831.json`、`resolved_d_20260831.json`：P/D 真实模型配置通过 `resolve_once()` 后处理；保持 BF16 KV、8K chunk 和 FlashInfer。Kimi 投机验证自动选择 `nv_cutedsl`，P 侧 PP 自动关闭 overlap scheduler；该检查不加载模型权重。 -- `sm120_cutlass_compile_20260831.log`：SM120 预编译进度。首次构建被外部 Docker SIGKILL 终止，退出 137；Docker 事件为显式 kill，无 OOM 事件。已编译的对象文件保留在独立缓存。 -- `sm120_cutlass_compile_resume_20260831.log`：恢复后编译成功，`SM120_CUTLASS_BUILD_OK`，退出 0。 -- `gpu_situ_smoke_detached_20260831.log`：首次完整单测发现 SiTU activation 白名单遗漏；已修复并增加 CPU 回归。 -- `gpu_situ_smoke_fix_20260831.log`：修复后 GPU 回归通过，1 passed；首次 JIT 在内的总用时 771.10 秒。 -- `image_build_activation_fix_20260831.log`：固化修复后的镜像构建及 22 项 CPU 回归。 -- `final_image_gpu_verify_20260831.log`、`final_image_gpu_container_20260831.json`：最终镜像独立 GPU 复测、Mooncake 导入、镜像 ID 和挂载证据。 -- `pd-dflash-20260831-1617/logs/p_0.log` 至 `p_3.log`：首轮 P 预热报错原始证据；该 Run 未进入请求测试，退出码 1。 -- `image_build_pwarm1_20260831.log`：预热修复镜像构建及 23 项 CPU 回归。 -- `pwarm1_602_cpu.log`、`pwarm1_602_inspect.json`：约 49 MiB 增量包导入后的容器回归与平台摘要。 -- `pd-dflash-pwarm1-20260831-1652/logs/p_0.log`：P 预热完成后，在初始化传输管理器时发现 `HybridLinearKVPool.end_layer` 缺失；未进入 D 启动和请求测试。 -- `image_build_kvbounds1_20260831.log`、`kvbounds_image_context/`：复用原 FlashInfer 层、只修改 Python 的构建配方及 24 项回归。 - -已通过 compileall、Black 和 Ruff 的未定义变量/语法检查。 - -## 实验入口 - -代码位于 601 的 `/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation`。只运行 `deploy_pd_dflash.sh`，它负责 P/D 启动、健康检查、Router、smoke 和评测。`bench_gsm8k_acceptance.py` 沿用原客户端的 prompt 与请求设置，将结果标签改为 DFlash，并增加原始 `meta_info`、verify 次数和输出结束原因归档。 +唯一入口在 **601**；602-608 不需要手工再启动入口。配置见 [config.env](config.env)，请求由 [bench_gsm8k_acceptance.py](bench_gsm8k_acceptance.py) 生成。维护既有 GitLab 分支 `hzy-kimi3-pd-pp8-standard`。 ```bash -# 601：命令检查，不需要 GPU 或 sudo cd /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation -DRY_RUN=1 RUN_ID=command-audit-20260831 bash deploy_pd_dflash.sh all +# 展开命令，不使用 GPU +DRY_RUN=1 RUN_ID=pd-dflash-command-audit bash deploy_pd_dflash.sh all +# 新实验：八节点空闲后，入口负责启动、smoke、C1/C8、归档及清理 +RUN_ID=pd-dflash-$(date +%Y%m%d-%H%M%S) bash deploy_pd_dflash.sh all ``` -此前两轮均在 P 侧启动阶段退出，日志已保存、失败容器已清理。修复镜像同步完成后使用新 Run ID 重试。正式顺序为 `start` → `smoke` → `bench` → `logs` → `stop`，也可用 `all` 串行执行。所有操作使用同一个 `RUN_ID`。失败时保留本任务容器和日志，入口不会自动杀其他实验。 +本次实际分两段执行，使用同一个 Run ID： -配置选择：P 为 TP4/PP8/EP4，D 为 TP32/PP1/EP4，两侧 FlashInfer；KV 为 BF16，chunk=8192，page=64。为 C1/C8 评测将活跃请求与 Decode Graph 上限设为 8；未额外改变模型 context 上限。 +```bash +RUN_ID=pd-dflash-kvbounds1-20260831-1740 bash deploy_pd_dflash.sh all +# 服务和 smoke 成功后，客户端数据集路径报错。修正挂载，不重启服务： +RUN_ID=pd-dflash-kvbounds1-20260831-1740 bash deploy_pd_dflash.sh bench +RUN_ID=pd-dflash-kvbounds1-20260831-1740 bash deploy_pd_dflash.sh logs +RUN_ID=pd-dflash-kvbounds1-20260831-1740 bash deploy_pd_dflash.sh stop +``` -GSM8K 为原数据集前 64 题，沿用历史客户端的 5-shot、temperature=0、输出上限 512，分别运行 C1/C8。输入不裁剪。历史 5-shot 也取自测试集前 5 题，和本次 64 题有重叠：结果适用于与历史流程的部署回归，不作为独立无泄漏的模型准确率评估。逐题输出和截断情况保留。 +`bench` 只能复用已运行且配置匹配的服务；当前服务已清理，复测应使用新 Run ID 的 `all`。正式执行用 tmux 托管同一入口，没有新增 launch 脚本。 -## 后续 +## 6. 原始结果与交付 -确认 8 节点模型、镜像与网络一致，然后运行 C1 smoke 和固定 GSM8K 64 题 C1/C8。保留逐题答案、截断、接受直方图、总耗时、吞吐、原始 `meta_info` 与服务日志；不重跑 EAGLE3/no-spec 基线。当前客户端为非流式请求，不把客户端总耗时当作 TTFT 或 ITL。 +601 原始目录： -运行时设置 `SGLANG_CACHE_DIR=/cache`，FlashInfer、Triton、PyTorch 扩展和 CUDA 编译缓存也指向该挂载目录，`TMPDIR=/cache/tmp`，宿主路径见 `config.env` 的 `JIT_CACHE`，避免容器可写层占满根盘；预检会创建所需临时目录。 +```text +/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/ + results/pd-dflash-kvbounds1-20260831-1740/ + bench/gsm8k_c1.jsonl # 逐题输出及原始 meta_info + bench/gsm8k_c8.jsonl + bench/gsm8k_c1_summary.json + bench/gsm8k_c8_summary.json + commands/ # 8 条 docker run、Router、2 条 bench + logs/p_0.log ... p_3.log # P 四节点 + logs/d_0.log ... d_3.log # D 四节点 + logs/router.log + metadata/post_run_resources.txt # 八节点清理核验 + result_validation.json # 独立答案/统计复核及证据 SHA256 +``` -## 首轮启动修复 +本地副本：`/Users/hzy/Desktop/infra/docs/evidence/kimi_k3_pd_dflash/results/pd-dflash-kvbounds1-20260831-1740/`。轻量结果、完整服务日志与补丁随 GitLab 提交；模型与约 14GB OCI 镜像归档保留服务器，不纳入普通 Git。 -`base_runner.py::_dummy_run` 已将 PD Prefill target 设为普通 Decode 预热，但随后仍创建 `DFlashVerifyInput`。普通 Triton Attention 读取 `kv_indptr` 时因此报 `AttributeError`。修复让该分支的 `spec_info=None`，D 侧真正的 TARGET_VERIFY 路径不变。没有强行添加字段，也没有修改 DFlash 接受算法。 - -第二轮通过预热后，`PrefillBootstrapQueue._init_kv_manager` 直接访问混合池不存在的 `end_layer`。修复以现有的完整注意力层偏移加 `full_layer_nums` 得到传输范围，KDA 状态和 draft KV 仍分别使用明确的全局层 ID。新增回归覆盖 MLA/MHA 混合池、空完整注意力 stage、普通池和 layer-shard 分支，不将 K/V 两组条目误计为两倍层数。 - -镜像同步须检查 Docker 所在根盘，而不只检查模型盘 `/data`。本镜像层展开约 34.4GB，压缩内容约 15GB；共享层会减少增量占用。607 已按用户授权删除两个无容器引用的 vLLM 镜像，根盘恢复到约 107GB，原镜像元数据保存在 `results/607_vllm_images_before_delete_20260831.json`。用户清理 606 后其根盘恢复到约 91GB。601 使用 `ctr images export` 将既有 OCI 压缩层直接导出到 `/data`，其旧 vLLM 镜像尚未删除。 +首段 `exit_code=1` 如实保留，对应容器内数据集 `../` 路径错误；修复后 `bench_resume_exit_code=0`、`cleanup_exit_code=0`。失败客户端命令和日志另存为 `gsm8k_c1.dataset_path_failure.*`。此前 P 预热和混合池层范围两次失败保留独立 Run，未混入本轮评测结果。 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/deploy_pd_dflash.sh b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/deploy_pd_dflash.sh index 32f16d9..6846bfc 100644 --- a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/deploy_pd_dflash.sh +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/deploy_pd_dflash.sh @@ -210,9 +210,9 @@ bench() { local concurrency for concurrency in 1 8; do emit "gsm8k_c${concurrency}" "${P_NODES[0]}" docker run --rm --network host \ - -v "${GSM8K}:${GSM8K}:ro" -v "${BENCH_CLIENT}:/bench.py:ro" \ + -v "${GSM8K}:/dataset/gsm8k.jsonl:ro" -v "${BENCH_CLIENT}:/bench.py:ro" \ -v "${RESULT_ROOT}:/results" --entrypoint python3 "$PD_IMAGE" /bench.py \ - --data-path "$GSM8K" --num-questions 64 --num-shots 5 \ + --data-path /dataset/gsm8k.jsonl --num-questions 64 --num-shots 5 \ --max-new-tokens 512 --temperature 0 --top-p 1 --parallel "$concurrency" \ --speculative-algorithm DFLASH \ --host "${P_NODES[0]}" --port "$ROUTER_PORT" --backend srt \ diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/cpu_regression_20260831_final.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/cpu_regression_20260831_final.log new file mode 100644 index 0000000..18ef398 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/cpu_regression_20260831_final.log @@ -0,0 +1,31 @@ +/tmp/uv-kimi-pd-cache/archive-v0/JtVqGBlr4_hQgwWtLUFzX/lib/python3.12/site-packages/torch/_subclasses/functional_tensor.py:368: UserWarning: Failed to initialize NumPy: No module named 'numpy' (Triggered internally at /Users/runner/work/pytorch/pytorch/torch/csrc/utils/tensor_numpy.cpp:84.) + cpu = _conversion_method_template(device=torch.device("cpu")) +test_capture_ownership_including_boundaries (test_dflash_pp_context.TestPPContext.test_capture_ownership_including_boundaries) ... ok +test_invalid_capture_configuration (test_dflash_pp_context.TestPPContext.test_invalid_capture_configuration) ... ok +test_kimi_boundary_uses_next_stage_weights (test_dflash_pp_context.TestPPContext.test_kimi_boundary_uses_next_stage_weights) ... ok +test_last_stage_without_capture_uses_incoming_context (test_dflash_pp_context.TestPPContext.test_last_stage_without_capture_uses_incoming_context) ... ok +test_missing_incoming_context_fails_before_kv_write (test_dflash_pp_context.TestPPContext.test_missing_incoming_context_fails_before_kv_write) ... ok +test_nonfinal_pool_is_minimal_without_mutating_target_config (test_dflash_pp_context.TestPPContext.test_nonfinal_pool_is_minimal_without_mutating_target_config) ... ok +test_only_prefill_skips_draft_graph_initialization (test_dflash_pp_context.TestPPContext.test_only_prefill_skips_draft_graph_initialization) ... ok +test_pd_input_builder_and_spec_dispatch (test_dflash_pp_context.TestPPContext.test_pd_input_builder_and_spec_dispatch) ... ok +test_pp8_boundary_capture (test_dflash_pp_context.TestPPContext.test_pp8_boundary_capture) ... ok +test_pp8_projection_empty_stages_float32_and_bf16 (test_dflash_pp_context.TestPPContext.test_pp8_projection_empty_stages_float32_and_bf16) ... ok +test_pp_proxy_is_consumed_once_even_on_forward_failure (test_dflash_pp_context.TestPPContext.test_pp_proxy_is_consumed_once_even_on_forward_failure) ... ok +test_prefill_dummy_forward_has_no_verify_metadata (test_dflash_pp_context.TestPPContext.test_prefill_dummy_forward_has_no_verify_metadata) ... ok +test_prefill_transfer_bounds_for_hybrid_and_ordinary_pools (test_dflash_pp_context.TestPPContext.test_prefill_transfer_bounds_for_hybrid_and_ordinary_pools) ... ok +test_activation_selection_accepts_situ_and_legacy_enum (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_activation_selection_accepts_situ_and_legacy_enum) ... ok +test_capability_requires_public_parameters (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_capability_requires_public_parameters) ... ok +test_runner_preserves_swiglu_step_and_old_api (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_runner_preserves_swiglu_step_and_old_api) ... ok +test_weight_layout_and_situ_parameters (test_flashinfer_kimi_merge.TestFlashInferKimiMerge.test_weight_layout_and_situ_parameters) ... ok +test_all_gqa_shards_and_replicas_byte_exact (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_all_gqa_shards_and_replicas_byte_exact) ... ok +test_flat_stride_mismatch_rejected (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_flat_stride_mismatch_rejected) ... ok +test_invalid_metadata_rejected_before_copy (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_invalid_metadata_rejected_before_copy) ... ok +test_mooncake_mixed_entries_match_layers_before_transfer (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_mooncake_mixed_entries_match_layers_before_transfer) ... ok +test_real_kimi_dflash_page64_tp4_to_tp32 (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_real_kimi_dflash_page64_tp4_to_tp32) ... ok +test_replicated_mla_is_flat_and_coalesced (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_replicated_mla_is_flat_and_coalesced) ... ok +test_wire_roundtrip (test_mixed_kv_entry_layout.TestMixedKVEntryLayout.test_wire_roundtrip) ... ok + +---------------------------------------------------------------------- +Ran 24 tests in 0.424s + +OK diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_container_20260831.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_container_20260831.json new file mode 100644 index 0000000..3a10dec --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_container_20260831.json @@ -0,0 +1,427 @@ +[ + { + "Id": "ade3678d0340e6eb81b6149c3dd167f85690c009cb3c24f6b8ed61c99529a5e9", + "Created": "2026-08-31T06:09:44.850021061Z", + "Path": "python3", + "Args": [ + "-c", + "import mooncake.engine; print(\"MOONCAKE_ENGINE_IMPORT_OK\", flush=True); import pytest; raise SystemExit(pytest.main([\"-v\", \"-s\", \"/test_kimi.py::test_kimi_k3_sm120_situ_layout_and_noncontiguous_input\"]))" + ], + "State": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-08-31T06:09:44.909236209Z", + "FinishedAt": "2026-08-31T06:10:13.37845969Z" + }, + "Image": "sha256:6f998f448dd3545423e2bd73000c8bce991b4dbc9be66aff329b978b303c5987", + "ResolvConfPath": "/var/lib/docker/containers/ade3678d0340e6eb81b6149c3dd167f85690c009cb3c24f6b8ed61c99529a5e9/resolv.conf", + "HostnamePath": "/var/lib/docker/containers/ade3678d0340e6eb81b6149c3dd167f85690c009cb3c24f6b8ed61c99529a5e9/hostname", + "HostsPath": "/var/lib/docker/containers/ade3678d0340e6eb81b6149c3dd167f85690c009cb3c24f6b8ed61c99529a5e9/hosts", + "LogPath": "/var/lib/docker/containers/ade3678d0340e6eb81b6149c3dd167f85690c009cb3c24f6b8ed61c99529a5e9/ade3678d0340e6eb81b6149c3dd167f85690c009cb3c24f6b8ed61c99529a5e9-json.log", + "Name": "/hzy_kimi_dflash_final_image_verify", + "RestartCount": 0, + "Driver": "overlayfs", + "Platform": "linux", + "MountLabel": "", + "ProcessLabel": "", + "AppArmorProfile": "docker-default", + "ExecIDs": null, + "HostConfig": { + "Binds": [ + "/data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache", + "/data/hzy/src/sglang-kimi-sm120-postsync-wt/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py:/test_kimi.py:ro" + ], + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "none", + "PortBindings": {}, + "RestartPolicy": { + "Name": "no", + "MaximumRetryCount": 0 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "ConsoleSize": [ + 0, + 0 + ], + "CapAdd": null, + "CapDrop": null, + "CgroupnsMode": "private", + "Dns": null, + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "host", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": false, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": [ + "label=disable" + ], + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "Isolation": "", + "CpuShares": 0, + "Memory": 0, + "NanoCpus": 0, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": [], + "BlkioDeviceReadBps": [], + "BlkioDeviceWriteBps": [], + "BlkioDeviceReadIOps": [], + "BlkioDeviceWriteIOps": [], + "CpuPeriod": 0, + "CpuQuota": 0, + "CpuRealtimePeriod": 0, + "CpuRealtimeRuntime": 0, + "CpusetCpus": "", + "CpusetMems": "", + "Devices": [], + "DeviceCgroupRules": null, + "DeviceRequests": [ + { + "Driver": "", + "Count": 0, + "DeviceIDs": [ + "6" + ], + "Capabilities": [ + [ + "gpu" + ] + ], + "Options": {} + } + ], + "MemoryReservation": 0, + "MemorySwap": 0, + "MemorySwappiness": null, + "OomKillDisable": null, + "PidsLimit": null, + "Ulimits": [], + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0, + "MaskedPaths": [ + "/proc/acpi", + "/proc/asound", + "/proc/interrupts", + "/proc/kcore", + "/proc/keys", + "/proc/latency_stats", + "/proc/sched_debug", + "/proc/scsi", + "/proc/timer_list", + "/proc/timer_stats", + "/sys/devices/virtual/powercap", + "/sys/firmware", + "/sys/devices/system/cpu/cpu0/thermal_throttle", + "/sys/devices/system/cpu/cpu1/thermal_throttle", + "/sys/devices/system/cpu/cpu2/thermal_throttle", + "/sys/devices/system/cpu/cpu3/thermal_throttle", + "/sys/devices/system/cpu/cpu4/thermal_throttle", + "/sys/devices/system/cpu/cpu5/thermal_throttle", + "/sys/devices/system/cpu/cpu6/thermal_throttle", + "/sys/devices/system/cpu/cpu7/thermal_throttle", + "/sys/devices/system/cpu/cpu8/thermal_throttle", + "/sys/devices/system/cpu/cpu9/thermal_throttle", + "/sys/devices/system/cpu/cpu10/thermal_throttle", + "/sys/devices/system/cpu/cpu11/thermal_throttle", + "/sys/devices/system/cpu/cpu12/thermal_throttle", + "/sys/devices/system/cpu/cpu13/thermal_throttle", + "/sys/devices/system/cpu/cpu14/thermal_throttle", + "/sys/devices/system/cpu/cpu15/thermal_throttle", + "/sys/devices/system/cpu/cpu16/thermal_throttle", + "/sys/devices/system/cpu/cpu17/thermal_throttle", + "/sys/devices/system/cpu/cpu18/thermal_throttle", + "/sys/devices/system/cpu/cpu19/thermal_throttle", + "/sys/devices/system/cpu/cpu20/thermal_throttle", + "/sys/devices/system/cpu/cpu21/thermal_throttle", + "/sys/devices/system/cpu/cpu22/thermal_throttle", + "/sys/devices/system/cpu/cpu23/thermal_throttle", + "/sys/devices/system/cpu/cpu24/thermal_throttle", + "/sys/devices/system/cpu/cpu25/thermal_throttle", + "/sys/devices/system/cpu/cpu26/thermal_throttle", + "/sys/devices/system/cpu/cpu27/thermal_throttle", + "/sys/devices/system/cpu/cpu28/thermal_throttle", + "/sys/devices/system/cpu/cpu29/thermal_throttle", + "/sys/devices/system/cpu/cpu30/thermal_throttle", + "/sys/devices/system/cpu/cpu31/thermal_throttle", + "/sys/devices/system/cpu/cpu32/thermal_throttle", + "/sys/devices/system/cpu/cpu33/thermal_throttle", + "/sys/devices/system/cpu/cpu34/thermal_throttle", + "/sys/devices/system/cpu/cpu35/thermal_throttle", + "/sys/devices/system/cpu/cpu36/thermal_throttle", + "/sys/devices/system/cpu/cpu37/thermal_throttle", + "/sys/devices/system/cpu/cpu38/thermal_throttle", + "/sys/devices/system/cpu/cpu39/thermal_throttle", + "/sys/devices/system/cpu/cpu40/thermal_throttle", + "/sys/devices/system/cpu/cpu41/thermal_throttle", + "/sys/devices/system/cpu/cpu42/thermal_throttle", + "/sys/devices/system/cpu/cpu43/thermal_throttle", + "/sys/devices/system/cpu/cpu44/thermal_throttle", + "/sys/devices/system/cpu/cpu45/thermal_throttle", + "/sys/devices/system/cpu/cpu46/thermal_throttle", + "/sys/devices/system/cpu/cpu47/thermal_throttle", + "/sys/devices/system/cpu/cpu48/thermal_throttle", + "/sys/devices/system/cpu/cpu49/thermal_throttle", + "/sys/devices/system/cpu/cpu50/thermal_throttle", + "/sys/devices/system/cpu/cpu51/thermal_throttle", + "/sys/devices/system/cpu/cpu52/thermal_throttle", + "/sys/devices/system/cpu/cpu53/thermal_throttle", + "/sys/devices/system/cpu/cpu54/thermal_throttle", + "/sys/devices/system/cpu/cpu55/thermal_throttle", + "/sys/devices/system/cpu/cpu56/thermal_throttle", + "/sys/devices/system/cpu/cpu57/thermal_throttle", + "/sys/devices/system/cpu/cpu58/thermal_throttle", + "/sys/devices/system/cpu/cpu59/thermal_throttle", + "/sys/devices/system/cpu/cpu60/thermal_throttle", + "/sys/devices/system/cpu/cpu61/thermal_throttle", + "/sys/devices/system/cpu/cpu62/thermal_throttle", + "/sys/devices/system/cpu/cpu63/thermal_throttle", + "/sys/devices/system/cpu/cpu64/thermal_throttle", + "/sys/devices/system/cpu/cpu65/thermal_throttle", + "/sys/devices/system/cpu/cpu66/thermal_throttle", + "/sys/devices/system/cpu/cpu67/thermal_throttle", + "/sys/devices/system/cpu/cpu68/thermal_throttle", + "/sys/devices/system/cpu/cpu69/thermal_throttle", + "/sys/devices/system/cpu/cpu70/thermal_throttle", + "/sys/devices/system/cpu/cpu71/thermal_throttle", + "/sys/devices/system/cpu/cpu72/thermal_throttle", + "/sys/devices/system/cpu/cpu73/thermal_throttle", + "/sys/devices/system/cpu/cpu74/thermal_throttle", + "/sys/devices/system/cpu/cpu75/thermal_throttle", + "/sys/devices/system/cpu/cpu76/thermal_throttle", + "/sys/devices/system/cpu/cpu77/thermal_throttle", + "/sys/devices/system/cpu/cpu78/thermal_throttle", + "/sys/devices/system/cpu/cpu79/thermal_throttle", + "/sys/devices/system/cpu/cpu80/thermal_throttle", + "/sys/devices/system/cpu/cpu81/thermal_throttle", + "/sys/devices/system/cpu/cpu82/thermal_throttle", + "/sys/devices/system/cpu/cpu83/thermal_throttle", + "/sys/devices/system/cpu/cpu84/thermal_throttle", + "/sys/devices/system/cpu/cpu85/thermal_throttle", + "/sys/devices/system/cpu/cpu86/thermal_throttle", + "/sys/devices/system/cpu/cpu87/thermal_throttle", + "/sys/devices/system/cpu/cpu88/thermal_throttle", + "/sys/devices/system/cpu/cpu89/thermal_throttle", + "/sys/devices/system/cpu/cpu90/thermal_throttle", + "/sys/devices/system/cpu/cpu91/thermal_throttle", + "/sys/devices/system/cpu/cpu92/thermal_throttle", + "/sys/devices/system/cpu/cpu93/thermal_throttle", + "/sys/devices/system/cpu/cpu94/thermal_throttle", + "/sys/devices/system/cpu/cpu95/thermal_throttle", + "/sys/devices/system/cpu/cpu96/thermal_throttle", + "/sys/devices/system/cpu/cpu97/thermal_throttle", + "/sys/devices/system/cpu/cpu98/thermal_throttle", + "/sys/devices/system/cpu/cpu99/thermal_throttle", + "/sys/devices/system/cpu/cpu100/thermal_throttle", + "/sys/devices/system/cpu/cpu101/thermal_throttle", + "/sys/devices/system/cpu/cpu102/thermal_throttle", + "/sys/devices/system/cpu/cpu103/thermal_throttle", + "/sys/devices/system/cpu/cpu104/thermal_throttle", + "/sys/devices/system/cpu/cpu105/thermal_throttle", + "/sys/devices/system/cpu/cpu106/thermal_throttle", + "/sys/devices/system/cpu/cpu107/thermal_throttle", + "/sys/devices/system/cpu/cpu108/thermal_throttle", + "/sys/devices/system/cpu/cpu109/thermal_throttle", + "/sys/devices/system/cpu/cpu110/thermal_throttle", + "/sys/devices/system/cpu/cpu111/thermal_throttle", + "/sys/devices/system/cpu/cpu112/thermal_throttle", + "/sys/devices/system/cpu/cpu113/thermal_throttle", + "/sys/devices/system/cpu/cpu114/thermal_throttle", + "/sys/devices/system/cpu/cpu115/thermal_throttle", + "/sys/devices/system/cpu/cpu116/thermal_throttle", + "/sys/devices/system/cpu/cpu117/thermal_throttle", + "/sys/devices/system/cpu/cpu118/thermal_throttle", + "/sys/devices/system/cpu/cpu119/thermal_throttle", + "/sys/devices/system/cpu/cpu120/thermal_throttle", + "/sys/devices/system/cpu/cpu121/thermal_throttle", + "/sys/devices/system/cpu/cpu122/thermal_throttle", + "/sys/devices/system/cpu/cpu123/thermal_throttle", + "/sys/devices/system/cpu/cpu124/thermal_throttle", + "/sys/devices/system/cpu/cpu125/thermal_throttle", + "/sys/devices/system/cpu/cpu126/thermal_throttle", + "/sys/devices/system/cpu/cpu127/thermal_throttle" + ], + "ReadonlyPaths": [ + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger" + ] + }, + "Storage": { + "RootFS": { + "Snapshot": { + "Name": "overlayfs" + } + } + }, + "Mounts": [ + { + "Type": "bind", + "Source": "/data/hzy/cache/kimi-dflash-fi0618-situ4460", + "Destination": "/cache", + "Mode": "", + "RW": true, + "Propagation": "rprivate" + }, + { + "Type": "bind", + "Source": "/data/hzy/src/sglang-kimi-sm120-postsync-wt/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py", + "Destination": "/test_kimi.py", + "Mode": "ro", + "RW": false, + "Propagation": "rprivate" + } + ], + "Config": { + "Hostname": "ade3678d0340", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": [ + "FLASHINFER_CUDA_ARCH_LIST=12.0f", + "MAX_JOBS=8", + "PYTHONUNBUFFERED=1", + "FLASHINFER_WORKSPACE_BASE=/cache", + "PATH=/root/.cargo/bin:/opt/sglang/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/nvidia/bin", + "NVARCH=x86_64", + "NVIDIA_REQUIRE_CUDA=cuda>=13.0 brand=unknown,driver>=535,driver<536 brand=grid,driver>=535,driver<536 brand=tesla,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=vapps,driver>=535,driver<536 brand=vpc,driver>=535,driver<536 brand=vcs,driver>=535,driver<536 brand=vws,driver>=535,driver<536 brand=cloudgaming,driver>=535,driver<536 brand=unknown,driver>=550,driver<551 brand=grid,driver>=550,driver<551 brand=tesla,driver>=550,driver<551 brand=nvidia,driver>=550,driver<551 brand=quadro,driver>=550,driver<551 brand=quadrortx,driver>=550,driver<551 brand=nvidiartx,driver>=550,driver<551 brand=vapps,driver>=550,driver<551 brand=vpc,driver>=550,driver<551 brand=vcs,driver>=550,driver<551 brand=vws,driver>=550,driver<551 brand=cloudgaming,driver>=550,driver<551 brand=unknown,driver>=565,driver<566 brand=grid,driver>=565,driver<566 brand=tesla,driver>=565,driver<566 brand=nvidia,driver>=565,driver<566 brand=quadro,driver>=565,driver<566 brand=quadrortx,driver>=565,driver<566 brand=nvidiartx,driver>=565,driver<566 brand=vapps,driver>=565,driver<566 brand=vpc,driver>=565,driver<566 brand=vcs,driver>=565,driver<566 brand=vws,driver>=565,driver<566 brand=cloudgaming,driver>=565,driver<566 brand=unknown,driver>=570,driver<571 brand=grid,driver>=570,driver<571 brand=tesla,driver>=570,driver<571 brand=nvidia,driver>=570,driver<571 brand=quadro,driver>=570,driver<571 brand=quadrortx,driver>=570,driver<571 brand=nvidiartx,driver>=570,driver<571 brand=vapps,driver>=570,driver<571 brand=vpc,driver>=570,driver<571 brand=vcs,driver>=570,driver<571 brand=vws,driver>=570,driver<571 brand=cloudgaming,driver>=570,driver<571 brand=unknown,driver>=575,driver<576 brand=grid,driver>=575,driver<576 brand=tesla,driver>=575,driver<576 brand=nvidia,driver>=575,driver<576 brand=quadro,driver>=575,driver<576 brand=quadrortx,driver>=575,driver<576 brand=nvidiartx,driver>=575,driver<576 brand=vapps,driver>=575,driver<576 brand=vpc,driver>=575,driver<576 brand=vcs,driver>=575,driver<576 brand=vws,driver>=575,driver<576 brand=cloudgaming,driver>=575,driver<576", + "NV_CUDA_CUDART_VERSION=13.0.96-1", + "CUDA_VERSION=13.0.3", + "LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64", + "NVIDIA_VISIBLE_DEVICES=all", + "NVIDIA_DRIVER_CAPABILITIES=compute,utility", + "NV_CUDA_LIB_VERSION=13.0.3-1", + "NV_NVTX_VERSION=13.0.85-1", + "NV_LIBNPP_VERSION=13.0.1.2-1", + "NV_LIBNPP_PACKAGE=libnpp-13-0=13.0.1.2-1", + "NV_LIBCUSPARSE_VERSION=12.6.3.3-1", + "NV_LIBCUBLAS_PACKAGE_NAME=libcublas-13-0", + "NV_LIBCUBLAS_VERSION=13.1.1.3-1", + "NV_LIBCUBLAS_PACKAGE=libcublas-13-0=13.1.1.3-1", + "NV_LIBNCCL_PACKAGE_NAME=libnccl2", + "NV_LIBNCCL_PACKAGE_VERSION=2.28.3-1", + "NCCL_VERSION=2.28.3-1", + "NV_LIBNCCL_PACKAGE=libnccl2=2.28.3-1+cuda13.0", + "NVIDIA_PRODUCT_NAME=CUDA", + "NV_CUDA_CUDART_DEV_VERSION=13.0.96-1", + "NV_NVML_DEV_VERSION=13.0.87-1", + "NV_LIBCUSPARSE_DEV_VERSION=12.6.3.3-1", + "NV_LIBNPP_DEV_VERSION=13.0.1.2-1", + "NV_LIBNPP_DEV_PACKAGE=libnpp-dev-13-0=13.0.1.2-1", + "NV_LIBCUBLAS_DEV_VERSION=13.1.1.3-1", + "NV_LIBCUBLAS_DEV_PACKAGE_NAME=libcublas-dev-13-0", + "NV_LIBCUBLAS_DEV_PACKAGE=libcublas-dev-13-0=13.1.1.3-1", + "NV_CUDA_NSIGHT_COMPUTE_VERSION=13.0.3-1", + "NV_CUDA_NSIGHT_COMPUTE_DEV_PACKAGE=cuda-nsight-compute-13-0=13.0.3-1", + "NV_LIBNCCL_DEV_PACKAGE_NAME=libnccl-dev", + "NV_LIBNCCL_DEV_PACKAGE_VERSION=2.28.3-1", + "NV_LIBNCCL_DEV_PACKAGE=libnccl-dev=2.28.3-1+cuda13.0", + "LIBRARY_PATH=/usr/local/cuda/lib64/stubs", + "NV_CUDNN_VERSION=9.14.0.64-1", + "NV_CUDNN_PACKAGE_NAME=libcudnn9-cuda-13", + "NV_CUDNN_PACKAGE=libcudnn9-cuda-13=9.14.0.64-1", + "NV_CUDNN_PACKAGE_DEV=libcudnn9-dev-cuda-13=9.14.0.64-1", + "NV_CUDNN_PACKAGE_DEV_HEADERS=libcudnn9-headers-cuda-13=9.14.0.64-1", + "DEBIAN_FRONTEND=noninteractive", + "CUDA_HOME=/usr/local/cuda", + "GDRCOPY_HOME=/usr/src/gdrdrv-2.5.1/", + "FLASHINFER_VERSION=0.6.17", + "LANG=en_US.UTF-8", + "LANGUAGE=en_US:en", + "LC_ALL=en_US.UTF-8", + "SGLANG_BUILD_COMMIT=daf631719690e18d13f67a20eb513fd48c712327", + "SGLANG_BUILD_URL=https://github.com/sgl-project/sglang/actions/runs/33137164725", + "SGLANG_IMAGE_TAG=lmsysorg/sglang:nightly-dev-20260828-daf63171", + "PYTHONPATH=/opt/kimi-dflash/python", + "PYTHONDONTWRITEBYTECODE=1", + "FLASHINFER_DISABLE_VERSION_CHECK=", + "SGLANG_SOURCE_ROOT=/opt/kimi-dflash" + ], + "Cmd": [ + "-c", + "import mooncake.engine; print(\"MOONCAKE_ENGINE_IMPORT_OK\", flush=True); import pytest; raise SystemExit(pytest.main([\"-v\", \"-s\", \"/test_kimi.py::test_kimi_k3_sm120_situ_layout_and_noncontiguous_input\"]))" + ], + "Image": "local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460", + "Volumes": null, + "WorkingDir": "/sgl-workspace/sglang", + "Entrypoint": [ + "python3" + ], + "Labels": { + "ai.sglang.build.commit": "daf631719690e18d13f67a20eb513fd48c712327", + "ai.sglang.build.url": "https://github.com/sgl-project/sglang/actions/runs/33137164725", + "ai.sglang.image.tag": "lmsysorg/sglang:nightly-dev-20260828-daf63171", + "com.nvidia.cudnn.version": "9.14.0.64-1", + "maintainer": "NVIDIA CORPORATION ", + "org.opencontainers.image.revision": "daf631719690e18d13f67a20eb513fd48c712327", + "org.opencontainers.image.source": "https://github.com/sgl-project/sglang", + "org.opencontainers.image.url": "https://github.com/sgl-project/sglang/actions/runs/33137164725", + "org.opencontainers.image.version": "lmsysorg/sglang:nightly-dev-20260828-daf63171" + } + }, + "NetworkSettings": { + "SandboxID": "", + "SandboxKey": "", + "Ports": {}, + "Networks": { + "none": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "DriverOpts": null, + "GwPriority": 0, + "NetworkID": "13a7d53fe7d1eb2261fa2476b4a017a554e9d1341e28d1fa6b6b39c57231ed2d", + "EndpointID": "", + "Gateway": "", + "IPAddress": "", + "MacAddress": "", + "IPPrefixLen": 0, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "DNSNames": null + } + } + }, + "ImageManifestDescriptor": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:14dc45076169180f94715109022fa5ba8116bdc68b5394ebf9c1801640aa04aa", + "size": 15020, + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + } +] diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_verify_20260831.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_verify_20260831.log new file mode 100644 index 0000000..a7e1da6 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/final_image_gpu_verify_20260831.log @@ -0,0 +1,18 @@ +MOONCAKE_ENGINE_IMPORT_OK +============================= test session starts ============================== +platform linux -- Python 3.12.3, pytest-9.1.1, pluggy-1.6.0 -- /opt/sglang/bin/python3 +cachedir: .pytest_cache +rootdir: / +plugins: anyio-4.14.2, typeguard-4.6.0 +collecting ... collected 1 item + +../../test_kimi.py::test_kimi_k3_sm120_situ_layout_and_noncontiguous_input [TensorRT-LLM][INFO] Set logger level to INFO +PASSED + +=============================== warnings summary =============================== +test_kimi.py: 14 warnings + /opt/sglang/lib/python3.12/site-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +======================= 1 passed, 14 warnings in 25.28s ======================== diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.dataset_path_failure.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.dataset_path_failure.log new file mode 100644 index 0000000..b7e8435 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.dataset_path_failure.log @@ -0,0 +1,16 @@ +gsm8k_c1 174.1.60.1: docker run --rm --network host -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/../../../datasets/gsm8k/test.jsonl:/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/../../../datasets/gsm8k/test.jsonl:ro -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/bench_gsm8k_acceptance.py:/bench.py:ro -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740:/results --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 /bench.py --data-path /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/../../../datasets/gsm8k/test.jsonl --num-questions 64 --num-shots 5 --max-new-tokens 512 --temperature 0 --top-p 1 --parallel 1 --speculative-algorithm DFLASH --host 174.1.60.1 --port 31000 --backend srt --output-file /results/bench/gsm8k_c1.jsonl --summary-file /results/bench/gsm8k_c1_summary.json +/opt/kimi-dflash/python/sglang/srt/layers/quantization/awq/awq.py:53: UserWarning: Only CUDA, HIP and XPU support AWQ currently. + warnings.warn(f"Only CUDA, HIP and XPU support AWQ currently.") +/opt/kimi-dflash/python/sglang/srt/layers/quantization/gguf.py:69: UserWarning: Only CUDA, MUSA and NPU support GGUF quantization currently. + warnings.warn(f"Only CUDA, MUSA and NPU support GGUF quantization currently.") +Traceback (most recent call last): + File "/bench.py", line 204, in + main(args) + File "/bench.py", line 57, in main + lines = list(read_jsonl(args.data_path)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/kimi-dflash/python/sglang/utils.py", line 137, in read_jsonl + with open(filename) as fin: + ^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: '/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/../../../datasets/gsm8k/test.jsonl' +FAILED rc=1; logs: /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740 (containers retained for diagnosis) diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.jsonl new file mode 100644 index 0000000..f2ca199 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.jsonl @@ -0,0 +1,64 @@ +{"prompt_id": 0, "output": " Janet sells 16 - 3 - 4 = <<16-3-4=9>>9 duck eggs a day.\nShe makes 9 * 2 = $<<9*2=18>>18 every day at the farmer’s market.\n#### 18\n\n", "correct": true, "error": null, "completion_tokens": 56, "spec_accept_length": 14.0, "spec_accept_length_from_histogram": 16.0, "spec_accept_rate": 1.0, "spec_accepted_drafts": 60, "spec_proposed_drafts": 60, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "spec_verify_ct": 4, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "37f7d04630254733bbc17eb3d897dbad", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 730, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170792.4341302, "queue_time": 0.0005555152893066406, "reasoning_tokens": 0, "completion_tokens": 56, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 56}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.0575672248378396, "spec_accept_rate": 1.0, "spec_accept_length": 14.0, "spec_num_correct_drafts": 60, "spec_num_proposed_drafts": 60, "spec_verify_ct": 4, "spec_accepted_drafts": 60, "spec_proposed_drafts": 60, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "request_received_ts": 1788170790.7995293, "api_server_dispatch_finish_ts": 1788170790.8062823, "request_finished_ts": 1788170792.8570967, "decode_throughput": 9868950.588235294, "response_sent_to_client_ts": 1788170792.857933}} +{"prompt_id": 1, "output": " It takes 2/2=<<2/2=1>>1 bolt of white fiber\nSo the total amount of fabric is 2+1=<<2+1=3>>3 bolts of fabric\n#### 3\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 16.333333333333332, "spec_accept_length_from_histogram": 16.0, "spec_accept_rate": 1.0, "spec_accepted_drafts": 45, "spec_proposed_drafts": 45, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], "spec_verify_ct": 3, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "ad3037fbf5a44d47adf9454dec639530", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 694, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170794.2979336, "queue_time": 0.00025272369384765625, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 1.7520954250358045, "spec_accept_rate": 1.0, "spec_accept_length": 16.333333333333332, "spec_num_correct_drafts": 45, "spec_num_proposed_drafts": 45, "spec_verify_ct": 3, "spec_accepted_drafts": 45, "spec_proposed_drafts": 45, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], "request_received_ts": 1788170792.8703833, "api_server_dispatch_finish_ts": 1788170792.873931, "request_finished_ts": 1788170794.6224787, "decode_throughput": 12269874.432091417, "response_sent_to_client_ts": 1788170794.6230125}} +{"prompt_id": 2, "output": " The cost of the house and repairs came out to 80,000+50,000=$<<80000+50000=130000>>130,000\nHe increased the value of the house by 80,000*1.5=<<80000*1.5=120000>>120,000\nSo the new value of the house is 120,000+80,000=$<<120000+80000=200000>>200,000\nSo he made a profit of 200,000-130,000=$<<200000-130000=70000>>70,000\n#### 70000\n\n", "correct": true, "error": null, "completion_tokens": 131, "spec_accept_length": 14.555555555555555, "spec_accept_length_from_histogram": 14.666666666666666, "spec_accept_rate": 0.9111111111111111, "spec_accepted_drafts": 123, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c5133d42fece4d9ba2525ced4167d296", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 717, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170796.1166427, "queue_time": 0.0003952980041503906, "reasoning_tokens": 0, "completion_tokens": 131, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 131}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.386726868804544, "spec_accept_rate": 0.9111111111111111, "spec_accept_length": 14.555555555555555, "spec_num_correct_drafts": 123, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 123, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7], "request_received_ts": 1788170794.6340747, "api_server_dispatch_finish_ts": 1788170794.6374896, "request_finished_ts": 1788170797.0208015, "decode_throughput": 31081370.99087063, "response_sent_to_client_ts": 1788170797.0213184}} +{"prompt_id": 3, "output": " He sprints 3*3=<<3*3=9>>9 times\nSo he runs 9*60=<<9*60=540>>540 meters\n#### 540\n\n", "correct": true, "error": null, "completion_tokens": 41, "spec_accept_length": 13.666666666666666, "spec_accept_length_from_histogram": 14.0, "spec_accept_rate": 0.8666666666666667, "spec_accepted_drafts": 39, "spec_proposed_drafts": 45, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2], "spec_verify_ct": 3, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "eab08018cad64015aa70846692653714", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 702, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170798.4501424, "queue_time": 0.0004611015319824219, "reasoning_tokens": 0, "completion_tokens": 41, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 41}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 1.7492218469269574, "spec_accept_rate": 0.8666666666666667, "spec_accept_length": 13.666666666666666, "spec_num_correct_drafts": 39, "spec_num_proposed_drafts": 45, "spec_verify_ct": 3, "spec_accepted_drafts": 39, "spec_proposed_drafts": 45, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2], "request_received_ts": 1788170797.031961, "api_server_dispatch_finish_ts": 1788170797.0353174, "request_finished_ts": 1788170798.7811828, "decode_throughput": 9854232.63966961, "response_sent_to_client_ts": 1788170798.781778}} +{"prompt_id": 4, "output": " If each chicken eats 3 cups of feed per day, then for 20 chickens they would need 3*20=<<3*20=60>>60 cups of feed per day.\nIf she feeds the flock 15 cups of feed in the morning, and 25 cups in the afternoon, then the final meal would require 60-15-25=<<60-15-25=20>>20 cups of chicken feed.\n#### 20\n\n", "correct": true, "error": null, "completion_tokens": 94, "spec_accept_length": 15.666666666666666, "spec_accept_length_from_histogram": 16.0, "spec_accept_rate": 1.0, "spec_accepted_drafts": 90, "spec_proposed_drafts": 90, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "spec_verify_ct": 6, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3a2f9c8c906a4f3a82f133d1decf27f8", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 775, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170800.3156128, "queue_time": 0.0003247261047363281, "reasoning_tokens": 0, "completion_tokens": 94, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 94}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.1533947782590985, "spec_accept_rate": 1.0, "spec_accept_length": 15.666666666666666, "spec_num_correct_drafts": 90, "spec_num_proposed_drafts": 90, "spec_verify_ct": 6, "spec_accepted_drafts": 90, "spec_proposed_drafts": 90, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "request_received_ts": 1788170798.7940645, "api_server_dispatch_finish_ts": 1788170798.7976167, "request_finished_ts": 1788170800.9474595, "decode_throughput": 22012121.59858922, "response_sent_to_client_ts": 1788170800.9479864}} +{"prompt_id": 5, "output": " The discount price of one glass is 60/100 * 5 = $<<60/100*5=3>>3.\nIf every second glass is cheaper, that means Kylar is going to buy 16 / 2 = <<16/2=8>>8 cheaper glasses.\nSo for the cheaper glasses, Kylar is going to pay 8 * 3 = $<<8*3=24>>24.\nAnd for the regular-priced glasses, Kylar will pay 8 * 5 = $<<8*5=40>>40.\nSo in total Kylar needs to pay 24 + 40 = $<<24+40=64>>64 for the glasses he wants to buy.\n#### 64\n\n", "correct": true, "error": null, "completion_tokens": 150, "spec_accept_length": 7.142857142857143, "spec_accept_length_from_histogram": 7.190476190476191, "spec_accept_rate": 0.4126984126984127, "spec_accepted_drafts": 130, "spec_proposed_drafts": 315, "spec_accept_histogram": [0, 0, 3, 4, 4, 0, 1, 1, 2, 0, 2, 1, 3], "spec_verify_ct": 21, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "9f60e0e725b34cdbba4207dc3456bd5c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 720, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170802.37141, "queue_time": 0.00030422210693359375, "reasoning_tokens": 0, "completion_tokens": 150, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 150}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.544350159354508, "spec_accept_rate": 0.4126984126984127, "spec_accept_length": 7.142857142857143, "spec_num_correct_drafts": 130, "spec_num_proposed_drafts": 315, "spec_verify_ct": 21, "spec_accepted_drafts": 130, "spec_proposed_drafts": 315, "spec_correct_drafts_histogram": [0, 0, 3, 4, 4, 0, 1, 1, 2, 0, 2, 1, 3], "spec_accept_histogram": [0, 0, 3, 4, 4, 0, 1, 1, 2, 0, 2, 1, 3], "request_received_ts": 1788170800.9583046, "api_server_dispatch_finish_ts": 1788170800.961818, "request_finished_ts": 1788170804.5026548, "decode_throughput": 68856265.02087368, "response_sent_to_client_ts": 1788170804.5029614}} +{"prompt_id": 6, "output": " If Seattle has 20 sheep, Charleston has 4 * 20 sheep = <<20*4=80>>80 sheep\nToulouse has twice as many sheep as Charleston, which is 2 * 80 sheep = <<2*80=160>>160 sheep\nTogether, the three has 20 sheep + 160 sheep + 80 sheep = <<20+160+80=260>>260 sheep\n#### 260\n\n", "correct": true, "error": null, "completion_tokens": 90, "spec_accept_length": 6.428571428571429, "spec_accept_length_from_histogram": 6.428571428571429, "spec_accept_rate": 0.3619047619047619, "spec_accepted_drafts": 76, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 2, 3, 2, 0, 0, 2, 2, 0, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "67473c8a9abe4ca8bec80fcefac1cae2", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 709, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170805.9356039, "queue_time": 0.00034356117248535156, "reasoning_tokens": 0, "completion_tokens": 90, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 90}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.8471194826997817, "spec_accept_rate": 0.3619047619047619, "spec_accept_length": 6.428571428571429, "spec_num_correct_drafts": 76, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 76, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 2, 3, 2, 0, 0, 2, 2, 0, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 2, 3, 2, 0, 0, 2, 2, 0, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788170804.5125027, "api_server_dispatch_finish_ts": 1788170804.514273, "request_finished_ts": 1788170807.359622, "decode_throughput": 44928548.34790785, "response_sent_to_client_ts": 1788170807.359861}} +{"prompt_id": 7, "output": " First find how many gigabytes are in 40% of the file: 200 GB * 40% = <<200*40*.01=80>>80 GB\nThen divide that number by the download rate to find the time until Windows restarts: 80 GB / 2 GB/minute = <<80/2=40>>40 minutes\nThen find the time to download the whole file after the restart: 200 GB / 2 GB/minute = <<200/2=100>>100 minutes\nThen add the time to download 40% of the file, to download the whole file, and to wait for Windows to update: 40 minutes + 100 minutes + 20 minutes = <<40+100+20=160>>160 minutes\n#### 160\n\n", "correct": true, "error": null, "completion_tokens": 161, "spec_accept_length": 6.1923076923076925, "spec_accept_length_from_histogram": 6.1923076923076925, "spec_accept_rate": 0.34615384615384615, "spec_accepted_drafts": 135, "spec_proposed_drafts": 390, "spec_accept_histogram": [0, 1, 7, 5, 1, 3, 2, 1, 1, 1, 1, 0, 1, 1, 0, 1], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e94effbfd96940738fbd013388125977", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 733, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170808.787831, "queue_time": 0.00044083595275878906, "reasoning_tokens": 0, "completion_tokens": 161, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 161}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.029898695182055, "spec_accept_rate": 0.34615384615384615, "spec_accept_length": 6.1923076923076925, "spec_num_correct_drafts": 135, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 135, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [0, 1, 7, 5, 1, 3, 2, 1, 1, 1, 1, 0, 1, 1, 0, 1], "spec_accept_histogram": [0, 1, 7, 5, 1, 3, 2, 1, 1, 1, 1, 0, 1, 1, 0, 1], "request_received_ts": 1788170807.369362, "api_server_dispatch_finish_ts": 1788170807.3711884, "request_finished_ts": 1788170811.399261, "decode_throughput": 57611902.02548625, "response_sent_to_client_ts": 1788170811.3995519}} +{"prompt_id": 8, "output": " When he turned around he was 3*60=<<3*60=180>>180 miles from home\nHe was only able to drive 4-2=<<4-2=2>>2 hours in the first four hours\nIn half an hour he goes 30*.5=<<30*.5=15>>15 miles\nHe then drives another 2-.5=<<2-.5=1.5>>1.5 hours\nIn that time he goes 80*1.5=<<80*1.5=120>>120 miles\nSo he drove 120+15=<<120+15=135>>135 miles\nSo he is 180-135=<<180-135=45>>45 miles away from home\n#### 45\n\n", "correct": true, "error": null, "completion_tokens": 161, "spec_accept_length": 6.708333333333333, "spec_accept_length_from_histogram": 6.75, "spec_accept_rate": 0.38333333333333336, "spec_accepted_drafts": 138, "spec_proposed_drafts": 360, "spec_accept_histogram": [0, 1, 3, 3, 5, 3, 0, 2, 1, 3, 0, 1, 1, 0, 0, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "a2d7d1d506934d3aafd2dd1d5518ff71", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 769, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170812.9181814, "queue_time": 0.0003256797790527344, "reasoning_tokens": 0, "completion_tokens": 161, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 161}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.9094296619296074, "spec_accept_rate": 0.38333333333333336, "spec_accept_length": 6.708333333333333, "spec_num_correct_drafts": 138, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 138, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [0, 1, 3, 3, 5, 3, 0, 2, 1, 3, 0, 1, 1, 0, 0, 1], "spec_accept_histogram": [0, 1, 3, 3, 5, 3, 0, 2, 1, 3, 0, 1, 1, 0, 0, 1], "request_received_ts": 1788170811.4101784, "api_server_dispatch_finish_ts": 1788170811.4123018, "request_finished_ts": 1788170815.319608, "decode_throughput": 43859763.04314526, "response_sent_to_client_ts": 1788170815.3202214}} +{"prompt_id": 9, "output": " Eliza is entitled to 45 -40 = <<45-40=5>>5 hours overtime pay.\nHer hourly rate for the overtime pay is $10 x 1.2 = $<<10*1.2=12>>12.\nSo, Eliza will receive $12 x 5 =$<<12*5=60>>60 for overtime pay.\nHer regular weekly earning is $10 x 40 = $<<10*40=400>>400.\nThus, Eliza will receive a total of $400 + $60 = $<<400+60=460>>460 for this week's work.\n#### 460\n\n", "correct": true, "error": null, "completion_tokens": 130, "spec_accept_length": 5.416666666666667, "spec_accept_length_from_histogram": 5.541666666666667, "spec_accept_rate": 0.30277777777777776, "spec_accepted_drafts": 109, "spec_proposed_drafts": 360, "spec_accept_histogram": [0, 3, 4, 3, 2, 6, 1, 0, 3, 0, 1, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "693952583c9c4eb394926a7dade4d661", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 725, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170816.8394153, "queue_time": 0.00033354759216308594, "reasoning_tokens": 0, "completion_tokens": 130, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 130}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.938992687035352, "spec_accept_rate": 0.30277777777777776, "spec_accept_length": 5.416666666666667, "spec_num_correct_drafts": 109, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 109, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [0, 3, 4, 3, 2, 6, 1, 0, 3, 0, 1, 1], "spec_accept_histogram": [0, 3, 4, 3, 2, 6, 1, 0, 3, 0, 1, 1], "request_received_ts": 1788170815.331121, "api_server_dispatch_finish_ts": 1788170815.3345551, "request_finished_ts": 1788170819.2701137, "decode_throughput": 30876659.67365136, "response_sent_to_client_ts": 1788170819.2707355}} +{"prompt_id": 10, "output": " The number of downloads of the program in the second month increased to 3*60 = <<3*60=180>>180\nIn the first two months, the total number of downloads of the program was 180+60 = <<180+60=240>>240\nIn the third month, the number of downloads of the program reduced by 30/100*180 = <<30/100*180=54>>54\nThere were 180-54 = <<180-54=126>>126 downloads in the third month.\nIn the three months, the total number of downloads of the program was 126+240 = <<126+240=366>>366\n#### 366\n\n", "correct": true, "error": null, "completion_tokens": 142, "spec_accept_length": 6.454545454545454, "spec_accept_length_from_histogram": 6.5, "spec_accept_rate": 0.36666666666666664, "spec_accepted_drafts": 121, "spec_proposed_drafts": 330, "spec_accept_histogram": [0, 0, 1, 6, 1, 2, 6, 1, 2, 2, 1], "spec_verify_ct": 22, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "def23d431f4c454282ca128ab87ed164", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 726, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170820.7009501, "queue_time": 0.0004684925079345703, "reasoning_tokens": 0, "completion_tokens": 142, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 142}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.6686324710026383, "spec_accept_rate": 0.36666666666666664, "spec_accept_length": 6.454545454545454, "spec_num_correct_drafts": 121, "spec_num_proposed_drafts": 330, "spec_verify_ct": 22, "spec_accepted_drafts": 121, "spec_proposed_drafts": 330, "spec_correct_drafts_histogram": [0, 0, 1, 6, 1, 2, 6, 1, 2, 2, 1], "spec_accept_histogram": [0, 0, 1, 6, 1, 2, 6, 1, 2, 2, 1], "request_received_ts": 1788170819.281607, "api_server_dispatch_finish_ts": 1788170819.2849867, "request_finished_ts": 1788170822.9502394, "decode_throughput": 34392911.67287596, "response_sent_to_client_ts": 1788170822.9507558}} +{"prompt_id": 11, "output": " The total charge for the doughnuts was 3 x $68 = $<<3*68=204>>204.\nThe total charge for the mini cupcakes was 2 x $80 = $<<2*80=160>>160.\nThe total charge for the mini cheesecakes was 6 x $55 = $<<6*55=330>>330.\nTherefore the total amount Toula paid for the pastries was $204 + $160 + $330 = $<<204+160+330=694>>694.\n#### 694\n\n", "correct": true, "error": null, "completion_tokens": 111, "spec_accept_length": 7.928571428571429, "spec_accept_length_from_histogram": 8.0, "spec_accept_rate": 0.4666666666666667, "spec_accepted_drafts": 98, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 3, 1, 1, 1, 0, 0, 2, 0, 1, 1, 1, 1, 0, 0, 2], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "9f3a4eddf37b4df486b1c590a1b9c98c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 727, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170824.4234366, "queue_time": 0.000354766845703125, "reasoning_tokens": 0, "completion_tokens": 111, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 111}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.866735917981714, "spec_accept_rate": 0.4666666666666667, "spec_accept_length": 7.928571428571429, "spec_num_correct_drafts": 98, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 98, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 3, 1, 1, 1, 0, 0, 2, 0, 1, 1, 1, 1, 0, 0, 2], "spec_accept_histogram": [0, 3, 1, 1, 1, 0, 0, 2, 0, 1, 1, 1, 1, 0, 0, 2], "request_received_ts": 1788170822.9617724, "api_server_dispatch_finish_ts": 1788170822.9652433, "request_finished_ts": 1788170825.8285084, "decode_throughput": 1146.1920226028992, "response_sent_to_client_ts": 1788170825.8302448}} +{"prompt_id": 12, "output": " He makes $10.5 selling lemons each year because 7 x 1.5 = <<7*1.5=10.5>>10.5\nHe earns $7.5 each year from the lemon tree because 10.5 - 3 = <<10.5-3=7.5>>7.5\nIt will take 12 years to earn enough to pay off the tree because 90 / 7.5 = <<90/7.5=12>>12\nHe will make money in year 13 because 12 + 1 = <<12+1=13>>13\n#### 13\n\n", "correct": true, "error": null, "completion_tokens": 133, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 6.947368421052632, "spec_accept_rate": 0.39649122807017545, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 1, 4, 3, 2, 1, 1, 0, 0, 2, 2, 1, 1, 0, 0, 1], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "74583eeeeb86491ba2a992b05e9ecdf0", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 733, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170827.2494457, "queue_time": 0.00029158592224121094, "reasoning_tokens": 0, "completion_tokens": 133, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 133}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.313430523034185, "spec_accept_rate": 0.39649122807017545, "spec_accept_length": 7.0, "spec_num_correct_drafts": 113, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 1, 4, 3, 2, 1, 1, 0, 0, 2, 2, 1, 1, 0, 0, 1], "spec_accept_histogram": [0, 1, 4, 3, 2, 1, 1, 0, 0, 2, 2, 1, 1, 0, 0, 1], "request_received_ts": 1788170825.841031, "api_server_dispatch_finish_ts": 1788170825.8444462, "request_finished_ts": 1788170829.1544616, "decode_throughput": 35809479.72915614, "response_sent_to_client_ts": 1788170829.1549692}} +{"prompt_id": 13, "output": " First multiply the five remaining vacuum cleaners by two to find out how many Melanie had before she visited the orange house: 5 * 2 = <<5*2=10>>10\nThen add two to figure out how many vacuum cleaners she had before visiting the red house: 10 + 2 = <<10+2=12>>12\nNow we know that 2/3 * x = 12, where x is the number of vacuum cleaners Melanie started with. We can find x by dividing each side of the equation by 2/3, which produces x = 18\n#### 18\n\n", "correct": true, "error": null, "completion_tokens": 126, "spec_accept_length": 5.478260869565218, "spec_accept_length_from_histogram": 5.521739130434782, "spec_accept_rate": 0.30144927536231886, "spec_accepted_drafts": 104, "spec_proposed_drafts": 345, "spec_accept_histogram": [0, 2, 5, 3, 3, 2, 1, 3, 3, 0, 1], "spec_verify_ct": 23, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "39d8067b5c374c0c8e57222af97c20c1", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 725, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170830.595103, "queue_time": 0.00033545494079589844, "reasoning_tokens": 0, "completion_tokens": 126, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 126}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.7262040907517076, "spec_accept_rate": 0.30144927536231886, "spec_accept_length": 5.478260869565218, "spec_num_correct_drafts": 104, "spec_num_proposed_drafts": 345, "spec_verify_ct": 23, "spec_accepted_drafts": 104, "spec_proposed_drafts": 345, "spec_correct_drafts_histogram": [0, 2, 5, 3, 3, 2, 1, 3, 3, 0, 1], "spec_accept_histogram": [0, 2, 5, 3, 3, 2, 1, 3, 3, 0, 1], "request_received_ts": 1788170829.1658404, "api_server_dispatch_finish_ts": 1788170829.1692853, "request_finished_ts": 1788170832.8920445, "decode_throughput": 27877812.44158272, "response_sent_to_client_ts": 1788170832.8926399}} +{"prompt_id": 14, "output": " There are 20 x 20/100 = <<20*20/100=4>>4 students who enrolled in contemporary dance.\nSo, 20 - 4 = <<20-4=16>>16 students are enrolled in either jazz or hip-hop dance.\nThere are 16 x 25/100 = <<16*25/100=4>>4 students who enrolled in jazz dance.\nHence, 16 - 4 = <<16-4=12>>12 students enrolled in hip-hop dance.\nThis is 12/20 x 100% = 60% of the entire students.\n#### 60\n\n", "correct": true, "error": null, "completion_tokens": 128, "spec_accept_length": 6.7368421052631575, "spec_accept_length_from_histogram": 6.7894736842105265, "spec_accept_rate": 0.38596491228070173, "spec_accepted_drafts": 110, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 1, 4, 1, 1, 5, 0, 1, 0, 3, 0, 1, 2], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "a7730e48a01f470fa67425b4ca0859ab", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 717, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170834.349421, "queue_time": 0.0004432201385498047, "reasoning_tokens": 0, "completion_tokens": 128, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 128}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.3648715745657682, "spec_accept_rate": 0.38596491228070173, "spec_accept_length": 6.7368421052631575, "spec_num_correct_drafts": 110, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 110, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 1, 4, 1, 1, 5, 0, 1, 0, 3, 0, 1, 2], "spec_accept_histogram": [0, 1, 4, 1, 1, 5, 0, 1, 0, 3, 0, 1, 2], "request_received_ts": 1788170832.9033282, "api_server_dispatch_finish_ts": 1788170832.9066398, "request_finished_ts": 1788170836.2682, "decode_throughput": 33976631.779743366, "response_sent_to_client_ts": 1788170836.2732253}} +{"prompt_id": 15, "output": " If he purchases jewelry, he will make a profit of 2.5% which is $5000*(2.5/100) = $<<5000*(2.5/100)=125>>125\nIf he purchases electronic gadgets, he will make a profit of 1.2% which is $8000*(1.2/100) = $<<8000*(1.2/100)=96>>96\nIf he wants to maximize profit, since $125 > $96, he will choose to purchase jewelry, thereby making a profit of $<<125=125>>125\n#### 125\n\n", "correct": true, "error": null, "completion_tokens": 128, "spec_accept_length": 7.529411764705882, "spec_accept_length_from_histogram": 7.588235294117647, "spec_accept_rate": 0.4392156862745098, "spec_accepted_drafts": 112, "spec_proposed_drafts": 255, "spec_accept_histogram": [0, 1, 0, 4, 3, 1, 1, 2, 1, 0, 1, 0, 0, 0, 1, 2], "spec_verify_ct": 17, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "262436b409644a148400619cc4982392", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 758, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170837.7120037, "queue_time": 0.00043201446533203125, "reasoning_tokens": 0, "completion_tokens": 128, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 128}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.1382477888837457, "spec_accept_rate": 0.4392156862745098, "spec_accept_length": 7.529411764705882, "spec_num_correct_drafts": 112, "spec_num_proposed_drafts": 255, "spec_verify_ct": 17, "spec_accepted_drafts": 112, "spec_proposed_drafts": 255, "spec_correct_drafts_histogram": [0, 1, 0, 4, 3, 1, 1, 2, 1, 0, 1, 0, 0, 0, 1, 2], "spec_accept_histogram": [0, 1, 0, 4, 3, 1, 1, 2, 1, 0, 1, 0, 0, 0, 1, 2], "request_received_ts": 1788170836.2847843, "api_server_dispatch_finish_ts": 1788170836.288778, "request_finished_ts": 1788170839.4230323, "decode_throughput": 16817563.254362706, "response_sent_to_client_ts": 1788170839.4235678}} +{"prompt_id": 16, "output": " On the first day, the trains covered 2 trains * 80 miles/train = <<2*80=160>>160 miles together.\nThey also covered 150 miles/train * 2 trains = <<150*2=300>>300 miles together on the second day.\nThe combined distance the two trains covered in the two days is 300 miles + 160 miles = <<300+160=460>>460 miles\nThe average distance for the two days is 460 miles / 2 trains = <<460/2=230>>230 miles/train\n#### 230\n\n", "correct": true, "error": null, "completion_tokens": 117, "spec_accept_length": 5.85, "spec_accept_length_from_histogram": 5.9, "spec_accept_rate": 0.32666666666666666, "spec_accepted_drafts": 98, "spec_proposed_drafts": 300, "spec_accept_histogram": [0, 1, 3, 5, 3, 0, 0, 5, 0, 1, 2], "spec_verify_ct": 20, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "99c77fce13bc47df977defafe52793f4", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 717, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170840.8697152, "queue_time": 0.0003342628479003906, "reasoning_tokens": 0, "completion_tokens": 117, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 117}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.421430238056928, "spec_accept_rate": 0.32666666666666666, "spec_accept_length": 5.85, "spec_num_correct_drafts": 98, "spec_num_proposed_drafts": 300, "spec_verify_ct": 20, "spec_accepted_drafts": 98, "spec_proposed_drafts": 300, "spec_correct_drafts_histogram": [0, 1, 3, 5, 3, 0, 0, 5, 0, 1, 2], "spec_accept_histogram": [0, 1, 3, 5, 3, 0, 0, 5, 0, 1, 2], "request_received_ts": 1788170839.4343305, "api_server_dispatch_finish_ts": 1788170839.4378765, "request_finished_ts": 1788170842.8557606, "decode_throughput": 29546685.22927292, "response_sent_to_client_ts": 1788170842.8563435}} +{"prompt_id": 17, "output": " First find the total amount Jill makes per week teaching: $20/hour * 35 hours/week = $<<20*35=700>>700/week\nThen find the total amount Jill makes per week coaching: $30/hour * 15 hours/week = $<<30*15=450>>450/week\nThen add those two amounts to find the total amount Jill makes per week: $700/week + $450/week = $<<700+450=1150>>1150/week\nThen multiply that number by the number of weeks Jill works in a year to find her annual salary: $1150/week * 50 weeks/year = $<<1150*50=57500>>57,500\n#### 57500\n\n", "correct": true, "error": null, "completion_tokens": 150, "spec_accept_length": 6.0, "spec_accept_length_from_histogram": 6.08, "spec_accept_rate": 0.33866666666666667, "spec_accepted_drafts": 127, "spec_proposed_drafts": 375, "spec_accept_histogram": [0, 1, 5, 4, 2, 3, 5, 1, 1, 0, 1, 0, 1, 0, 1], "spec_verify_ct": 25, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "47468684c3f040aca3fe737647943ddb", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 719, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170844.3002982, "queue_time": 0.0002906322479248047, "reasoning_tokens": 0, "completion_tokens": 150, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 150}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.931107017211616, "spec_accept_rate": 0.33866666666666667, "spec_accept_length": 6.0, "spec_num_correct_drafts": 127, "spec_num_proposed_drafts": 375, "spec_verify_ct": 25, "spec_accepted_drafts": 127, "spec_proposed_drafts": 375, "spec_correct_drafts_histogram": [0, 1, 5, 4, 2, 3, 5, 1, 1, 0, 1, 0, 1, 0, 1], "spec_accept_histogram": [0, 1, 5, 4, 2, 3, 5, 1, 1, 0, 1, 0, 1, 0, 1], "request_received_ts": 1788170842.867236, "api_server_dispatch_finish_ts": 1788170842.8706605, "request_finished_ts": 1788170846.798343, "decode_throughput": 33678040.58014946, "response_sent_to_client_ts": 1788170846.7989378}} +{"prompt_id": 18, "output": " She eats 3 eggs every day and there are 7 days in a week so she eats 3*7 = <<3*7=21>>21 eggs a week\nAfter 4 weeks she will have eaten 4*21 = <<4*21=84>>84 eggs\nThere are 12 eggs in 1 dozen and she'll eat 84 eggs so that's 84/12 = <<84/12=7>>7 dozen eggs\n#### 7\n\n", "correct": true, "error": null, "completion_tokens": 98, "spec_accept_length": 6.125, "spec_accept_length_from_histogram": 6.1875, "spec_accept_rate": 0.3458333333333333, "spec_accepted_drafts": 83, "spec_proposed_drafts": 240, "spec_accept_histogram": [0, 1, 2, 3, 1, 5, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1], "spec_verify_ct": 16, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "67a3528939554615a843434a4d47c7f5", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 695, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170848.2723129, "queue_time": 0.0005748271942138672, "reasoning_tokens": 0, "completion_tokens": 98, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 98}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.0536309089511633, "spec_accept_rate": 0.3458333333333333, "spec_accept_length": 6.125, "spec_num_correct_drafts": 83, "spec_num_proposed_drafts": 240, "spec_verify_ct": 16, "spec_accepted_drafts": 83, "spec_proposed_drafts": 240, "spec_correct_drafts_histogram": [0, 1, 2, 3, 1, 5, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 3, 1, 5, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1], "request_received_ts": 1788170846.8097692, "api_server_dispatch_finish_ts": 1788170846.8130748, "request_finished_ts": 1788170849.8634002, "decode_throughput": 22454016.800258704, "response_sent_to_client_ts": 1788170849.8639147}} +{"prompt_id": 19, "output": " First figure out how many hours it takes to hike a 12-mile trail at 4 mph by dividing the distance by the speed: 12 miles / 4 mph = <<12/4=3>>3 hours\nNext subtract the time Marissa already spent walking to find out how much time she has left: 3 hours - 1 hour- 1 hour = <<3-1-1=1>>1 hour\nNow figure out how much distance she has left by subtracting the distance she already traveled from the total distance: 12 miles - 4 miles - 2 miles = <<12-4-2=6>>6 miles\nNow divide the remaining distance by the remaining time to find out how fast in miles per hour Marissa has to travel: 6 miles / 1 hour = <<6/1=6>>6 mph\n#### 6\n\n", "correct": true, "error": null, "completion_tokens": 181, "spec_accept_length": 5.65625, "spec_accept_length_from_histogram": 5.6875, "spec_accept_rate": 0.3125, "spec_accepted_drafts": 150, "spec_proposed_drafts": 480, "spec_accept_histogram": [1, 1, 2, 10, 4, 4, 3, 0, 3, 3, 1], "spec_verify_ct": 32, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5de38f47c87c48d5941172703a124b19", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 732, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170851.2814784, "queue_time": 0.00037932395935058594, "reasoning_tokens": 0, "completion_tokens": 181, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 181}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.63415600406006, "spec_accept_rate": 0.3125, "spec_accept_length": 5.65625, "spec_num_correct_drafts": 150, "spec_num_proposed_drafts": 480, "spec_verify_ct": 32, "spec_accepted_drafts": 150, "spec_proposed_drafts": 480, "spec_correct_drafts_histogram": [1, 1, 2, 10, 4, 4, 3, 0, 3, 3, 1], "spec_accept_histogram": [1, 1, 2, 10, 4, 4, 3, 0, 3, 3, 1], "request_received_ts": 1788170849.8748236, "api_server_dispatch_finish_ts": 1788170849.8781939, "request_finished_ts": 1788170854.5089796, "decode_throughput": 44822246.827458255, "response_sent_to_client_ts": 1788170854.5095549}} +{"prompt_id": 20, "output": " There are 15 x 3/5 = <<15*3/5=9>>9 liters of water from the 15 liters pineapple drink.\nAfter 1 liter of orange drink was spilled, there were 10 - 1 = <<10-1=9>>9 liters of orange drink left.\nOut of the 9 liters, 9 x 2/3 = <<9*2/3=6>>6 liters are water.\nThus, there are a total of 9 + 6 = <<9+6=15>>15 liters of water out of the 24 liters.\n#### 15\n\n", "correct": true, "error": null, "completion_tokens": 128, "spec_accept_length": 5.818181818181818, "spec_accept_length_from_histogram": 5.863636363636363, "spec_accept_rate": 0.3242424242424242, "spec_accepted_drafts": 107, "spec_proposed_drafts": 330, "spec_accept_histogram": [1, 1, 4, 3, 2, 6, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 22, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3d3b58f2ce5247739068430353295dc8", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170855.9673853, "queue_time": 0.0003261566162109375, "reasoning_tokens": 0, "completion_tokens": 128, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 128}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.6571107748895884, "spec_accept_rate": 0.3242424242424242, "spec_accept_length": 5.818181818181818, "spec_num_correct_drafts": 107, "spec_num_proposed_drafts": 330, "spec_verify_ct": 22, "spec_accepted_drafts": 107, "spec_proposed_drafts": 330, "spec_correct_drafts_histogram": [1, 1, 4, 3, 2, 6, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [1, 1, 4, 3, 2, 6, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788170854.5221355, "api_server_dispatch_finish_ts": 1788170854.5255566, "request_finished_ts": 1788170858.1792462, "decode_throughput": 105.74452865638081, "response_sent_to_client_ts": 1788170858.1798604}} +{"prompt_id": 21, "output": " When Raymond's son was born Samantha was 23 - 6 = <<23-6=17>>17 years old.\nThus it has been 31 - 17 = <<31-17=14>>14 years since Raymond's son was born.\n#### 14\n\n", "correct": true, "error": null, "completion_tokens": 56, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 7.125, "spec_accept_rate": 0.4083333333333333, "spec_accepted_drafts": 49, "spec_proposed_drafts": 120, "spec_accept_histogram": [0, 0, 0, 2, 0, 3, 0, 0, 2, 0, 0, 0, 1], "spec_verify_ct": 8, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "995465f2450c4564b3b909a13642a52f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 711, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170859.6130674, "queue_time": 0.00034308433532714844, "reasoning_tokens": 0, "completion_tokens": 56, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 56}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.2496120859868824, "spec_accept_rate": 0.4083333333333333, "spec_accept_length": 7.0, "spec_num_correct_drafts": 49, "spec_num_proposed_drafts": 120, "spec_verify_ct": 8, "spec_accepted_drafts": 49, "spec_proposed_drafts": 120, "spec_correct_drafts_histogram": [0, 0, 0, 2, 0, 3, 0, 0, 2, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 0, 2, 0, 3, 0, 0, 2, 0, 0, 0, 1], "request_received_ts": 1788170858.1908987, "api_server_dispatch_finish_ts": 1788170858.19429, "request_finished_ts": 1788170860.4405107, "decode_throughput": 14554725.8952557, "response_sent_to_client_ts": 1788170860.4410005}} +{"prompt_id": 22, "output": " His first 3 customers buy 3 * 1 = <<3*1=3>>3 DVDs.\nHis next 2 buy 2 * 2 = <<2*2=4>>4 DVDs.\nHe sells a total of 3 + 4 + 0 = <<3+4+0=7>>7 DVDs.\n#### 7\n\n", "correct": true, "error": null, "completion_tokens": 74, "spec_accept_length": 8.222222222222221, "spec_accept_length_from_histogram": 8.333333333333332, "spec_accept_rate": 0.4888888888888889, "spec_accepted_drafts": 66, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 0, 1, 1, 1, 0, 1, 0, 0, 3, 1, 0, 0, 0, 1], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3df38f96cfea4831b51639ef9f0d6dec", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 722, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170861.8394158, "queue_time": 0.0003330707550048828, "reasoning_tokens": 0, "completion_tokens": 74, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 74}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.297401685733348, "spec_accept_rate": 0.4888888888888889, "spec_accept_length": 8.222222222222221, "spec_num_correct_drafts": 66, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 66, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 0, 1, 1, 1, 0, 1, 0, 0, 3, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 1, 1, 0, 1, 0, 0, 3, 1, 0, 0, 0, 1], "request_received_ts": 1788170860.451573, "api_server_dispatch_finish_ts": 1788170860.4548833, "request_finished_ts": 1788170862.7489746, "decode_throughput": 21773098.097777776, "response_sent_to_client_ts": 1788170862.7495334}} +{"prompt_id": 23, "output": " The candle burns for 5 - 1 = <<5-1=4>>4 hours.\nThus, the candle will be 2 * 4 = <<2*4=8>>8 centimeters shorter.\n#### 8\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 6.125, "spec_accept_length_from_histogram": 6.25, "spec_accept_rate": 0.35, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_accept_histogram": [0, 0, 1, 1, 2, 1, 1, 0, 0, 2], "spec_verify_ct": 8, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "b069d0d44b624c119ebd3cbbf125058f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 706, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170864.1499302, "queue_time": 0.00030159950256347656, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.209783522877842, "spec_accept_rate": 0.35, "spec_accept_length": 6.125, "spec_num_correct_drafts": 42, "spec_num_proposed_drafts": 120, "spec_verify_ct": 8, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_correct_drafts_histogram": [0, 0, 1, 1, 2, 1, 1, 0, 0, 2], "spec_accept_histogram": [0, 0, 1, 1, 2, 1, 1, 0, 0, 2], "request_received_ts": 1788170862.7600243, "api_server_dispatch_finish_ts": 1788170862.7633939, "request_finished_ts": 1788170864.9698079, "decode_throughput": 13116072.668787377, "response_sent_to_client_ts": 1788170864.9703045}} +{"prompt_id": 24, "output": " Let X be the original price of the book. The discounted price is X - X*25% = $19.50.\nCombining like terms, we get .75X = $19.50\nDividing both sides by .75, we get X = $26.\n#### 26\n\n", "correct": true, "error": null, "completion_tokens": 63, "spec_accept_length": 5.25, "spec_accept_length_from_histogram": 5.166666666666667, "spec_accept_rate": 0.2777777777777778, "spec_accepted_drafts": 50, "spec_proposed_drafts": 180, "spec_accept_histogram": [1, 0, 2, 4, 1, 1, 1, 0, 1, 0, 0, 1], "spec_verify_ct": 12, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "1736ed4e30af4f4dad4bf92e5aaf7dcc", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 703, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170866.342023, "queue_time": 0.0008862018585205078, "reasoning_tokens": 0, "completion_tokens": 63, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 63}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.56409065797925, "spec_accept_rate": 0.2777777777777778, "spec_accept_length": 5.25, "spec_num_correct_drafts": 50, "spec_num_proposed_drafts": 180, "spec_verify_ct": 12, "spec_accepted_drafts": 50, "spec_proposed_drafts": 180, "spec_correct_drafts_histogram": [1, 0, 2, 4, 1, 1, 1, 0, 1, 0, 0, 1], "spec_accept_histogram": [1, 0, 2, 4, 1, 1, 1, 0, 1, 0, 0, 1], "request_received_ts": 1788170864.982442, "api_server_dispatch_finish_ts": 1788170864.9856894, "request_finished_ts": 1788170867.5465326, "decode_throughput": 17878875.54397744, "response_sent_to_client_ts": 1788170867.5470998}} +{"prompt_id": 25, "output": " Five packs of milk cost $3 x 5 = $<<3*5=15>>15.\nFour apples cost $1.50 x 4 = $<<1.5*4=6>>6.\nThe total cost of the chicken meal, milk, and apples is $12 + $15 + $6 = $<<12+15+6=33>>33.\nThus, the boxes of pizza cost $50 - $33 = $<<50-33=17>>17.\nTherefore, Marie ordered $17/$8.50 = <<17/8.5=2>>2 boxes of pizza.\n#### 2\n\n", "correct": true, "error": null, "completion_tokens": 131, "spec_accept_length": 6.894736842105263, "spec_accept_length_from_histogram": 6.947368421052632, "spec_accept_rate": 0.39649122807017545, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 1, 1, 2, 4, 2, 1, 3, 1, 1, 2, 0, 0, 0, 1], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5f79cc5908a944638013287169873acf", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 731, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170868.959407, "queue_time": 0.0003733634948730469, "reasoning_tokens": 0, "completion_tokens": 131, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 131}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.3248235899955034, "spec_accept_rate": 0.39649122807017545, "spec_accept_length": 6.894736842105263, "spec_num_correct_drafts": 113, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 1, 1, 2, 4, 2, 1, 3, 1, 1, 2, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 1, 2, 4, 2, 1, 3, 1, 1, 2, 0, 0, 0, 1], "request_received_ts": 1788170867.5586872, "api_server_dispatch_finish_ts": 1788170867.5619972, "request_finished_ts": 1788170870.8835108, "decode_throughput": 32499752.530849826, "response_sent_to_client_ts": 1788170870.884021}} +{"prompt_id": 26, "output": " 3 * (16.50 + 22.50 + 42) = $<<3*(16.50+22.50+42)=243>>243\nMishka spent $243 on clothing.\n#### 243\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 7.142857142857143, "spec_accept_rate": 0.4095238095238095, "spec_accepted_drafts": 43, "spec_proposed_drafts": 105, "spec_accept_histogram": [0, 1, 0, 0, 1, 0, 1, 2, 0, 2], "spec_verify_ct": 7, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "76e516be97654927966ce64f0496cd47", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 731, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170872.3097615, "queue_time": 0.00034809112548828125, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.130047915969044, "spec_accept_rate": 0.4095238095238095, "spec_accept_length": 7.0, "spec_num_correct_drafts": 43, "spec_num_proposed_drafts": 105, "spec_verify_ct": 7, "spec_accepted_drafts": 43, "spec_proposed_drafts": 105, "spec_correct_drafts_histogram": [0, 1, 0, 0, 1, 0, 1, 2, 0, 2], "spec_accept_histogram": [0, 1, 0, 0, 1, 0, 1, 2, 0, 2], "request_received_ts": 1788170870.8989816, "api_server_dispatch_finish_ts": 1788170870.9024105, "request_finished_ts": 1788170873.0290294, "decode_throughput": 12646204.772911299, "response_sent_to_client_ts": 1788170873.0296059}} +{"prompt_id": 27, "output": " Each container of ice cream has 15 servings and she eats 1 serving a night so after 60 days she will need 60/15 = 4 containers of ice cream\nIf each carton costs $4.00 and she needs 4 containers then it will cost her 4*4 = $<<4*4=16.00>>16.00\n#### 16\n\n", "correct": true, "error": null, "completion_tokens": 80, "spec_accept_length": 5.333333333333333, "spec_accept_length_from_histogram": 5.4, "spec_accept_rate": 0.29333333333333333, "spec_accepted_drafts": 66, "spec_proposed_drafts": 225, "spec_accept_histogram": [0, 1, 2, 3, 4, 2, 0, 2, 0, 0, 0, 0, 1], "spec_verify_ct": 15, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "6f1e8ef80f3c406cb2aee1be04249460", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 721, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170874.4416146, "queue_time": 0.0002911090850830078, "reasoning_tokens": 0, "completion_tokens": 80, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 80}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.9219643361866474, "spec_accept_rate": 0.29333333333333333, "spec_accept_length": 5.333333333333333, "spec_num_correct_drafts": 66, "spec_num_proposed_drafts": 225, "spec_verify_ct": 15, "spec_accepted_drafts": 66, "spec_proposed_drafts": 225, "spec_correct_drafts_histogram": [0, 1, 2, 3, 4, 2, 0, 2, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 3, 4, 2, 0, 2, 0, 0, 0, 0, 1], "request_received_ts": 1788170873.0405114, "api_server_dispatch_finish_ts": 1788170873.0438094, "request_finished_ts": 1788170875.9624758, "decode_throughput": 19749849.614901047, "response_sent_to_client_ts": 1788170875.9629645}} +{"prompt_id": 28, "output": " He traveled 20 miles + 15 miles = <<20+15=35>>35 miles not counting the distance between stops.\nHenry traveled 60 miles - 35 miles = <<60-35=25>>25 miles between his first and second stop.\n#### 25\n\n", "correct": true, "error": null, "completion_tokens": 57, "spec_accept_length": 5.7, "spec_accept_length_from_histogram": 5.8, "spec_accept_rate": 0.32, "spec_accepted_drafts": 48, "spec_proposed_drafts": 150, "spec_accept_histogram": [0, 2, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 10, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "1eebb668dd9843c29ac8824870f64a3a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 715, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170877.3851192, "queue_time": 0.0002970695495605469, "reasoning_tokens": 0, "completion_tokens": 57, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 57}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.4182498469017446, "spec_accept_rate": 0.32, "spec_accept_length": 5.7, "spec_num_correct_drafts": 48, "spec_num_proposed_drafts": 150, "spec_verify_ct": 10, "spec_accepted_drafts": 48, "spec_proposed_drafts": 150, "spec_correct_drafts_histogram": [0, 2, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 2, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788170875.9738064, "api_server_dispatch_finish_ts": 1788170875.9770806, "request_finished_ts": 1788170878.3920562, "decode_throughput": 14503025.119150989, "response_sent_to_client_ts": 1788170878.3926554}} +{"prompt_id": 29, "output": " The second pair of heels costs 33 * 2 = $<<33*2=66>>66.\nThe heels together cost 66 + 33 = $<<66+33=99>>99.\nThe boots cost $5 more than both pairs of heels together, so the boots cost 99 + 5 = $104.\n#### 104\n\n", "correct": true, "error": null, "completion_tokens": 73, "spec_accept_length": 6.636363636363637, "spec_accept_length_from_histogram": 6.7272727272727275, "spec_accept_rate": 0.38181818181818183, "spec_accepted_drafts": 63, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 1, 2, 1, 2, 1, 1, 0, 2, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e911caa130794ed0a7b436b4ce248e75", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 734, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170879.8008873, "queue_time": 0.0003352165222167969, "reasoning_tokens": 0, "completion_tokens": 73, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 73}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.5309368656016886, "spec_accept_rate": 0.38181818181818183, "spec_accept_length": 6.636363636363637, "spec_num_correct_drafts": 63, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 63, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 1, 2, 1, 2, 1, 1, 0, 2, 1], "spec_accept_histogram": [0, 0, 1, 2, 1, 2, 1, 1, 0, 2, 1], "request_received_ts": 1788170878.4037273, "api_server_dispatch_finish_ts": 1788170878.4071407, "request_finished_ts": 1788170880.9346642, "decode_throughput": 180.1111053272638, "response_sent_to_client_ts": 1788170880.9351585}} +{"prompt_id": 30, "output": " The total ratio representing their ages is 7+11= <<7+11=18>>18\nSince the fraction of the ratio that represents Allen's age is 11/18, Allen's current age is 11/18*162 = <<11/18*162=99>>99\nIf Allen is currently 99 years old, in 10 years he will be 99+10 = <<99+10=109>>109 years old\n#### 109\n\n", "correct": true, "error": null, "completion_tokens": 98, "spec_accept_length": 5.444444444444445, "spec_accept_length_from_histogram": 5.5, "spec_accept_rate": 0.3, "spec_accepted_drafts": 81, "spec_proposed_drafts": 270, "spec_accept_histogram": [0, 1, 4, 4, 1, 3, 1, 0, 3, 0, 0, 1], "spec_verify_ct": 18, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "837cec6d9f1542b9832531174d35dde4", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 703, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170882.3101165, "queue_time": 0.0002810955047607422, "reasoning_tokens": 0, "completion_tokens": 98, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 98}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.1735073472373188, "spec_accept_rate": 0.3, "spec_accept_length": 5.444444444444445, "spec_num_correct_drafts": 81, "spec_num_proposed_drafts": 270, "spec_verify_ct": 18, "spec_accepted_drafts": 81, "spec_proposed_drafts": 270, "spec_correct_drafts_histogram": [0, 1, 4, 4, 1, 3, 1, 0, 3, 0, 0, 1], "spec_accept_histogram": [0, 1, 4, 4, 1, 3, 1, 0, 3, 0, 0, 1], "request_received_ts": 1788170880.9458122, "api_server_dispatch_finish_ts": 1788170880.9491415, "request_finished_ts": 1788170884.1193197, "decode_throughput": 126.50049173681164, "response_sent_to_client_ts": 1788170884.1198432}} +{"prompt_id": 31, "output": " The second person guessed 60 because 20 + (80/2)= <<20+(80/2)=60>>60\nThe second guessed 100 because 80 x 1.25 = <<80*1.25=100>>100\nThe total amount guessed is 240 because 80 + 60 + 100= <<80+60+100=240>>240\nThe average amount guessed is 80 because 240 / 3 = <<240/3=80>>80\n#### 80\n\n", "correct": true, "error": null, "completion_tokens": 107, "spec_accept_length": 6.294117647058823, "spec_accept_length_from_histogram": 6.235294117647059, "spec_accept_rate": 0.34901960784313724, "spec_accepted_drafts": 89, "spec_proposed_drafts": 255, "spec_accept_histogram": [1, 2, 3, 3, 1, 0, 1, 2, 0, 1, 0, 1, 0, 1, 0, 1], "spec_verify_ct": 17, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "4cd11dbacf534e95b4048bc227108c12", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170885.5601656, "queue_time": 0.0004112720489501953, "reasoning_tokens": 0, "completion_tokens": 107, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 107}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.1245789229869843, "spec_accept_rate": 0.34901960784313724, "spec_accept_length": 6.294117647058823, "spec_num_correct_drafts": 89, "spec_num_proposed_drafts": 255, "spec_verify_ct": 17, "spec_accepted_drafts": 89, "spec_proposed_drafts": 255, "spec_correct_drafts_histogram": [1, 2, 3, 3, 1, 0, 1, 2, 0, 1, 0, 1, 0, 1, 0, 1], "spec_accept_histogram": [1, 2, 3, 3, 1, 0, 1, 2, 0, 1, 0, 1, 0, 1, 0, 1], "request_received_ts": 1788170884.1305885, "api_server_dispatch_finish_ts": 1788170884.1337116, "request_finished_ts": 1788170887.2551675, "decode_throughput": 30832082.71542733, "response_sent_to_client_ts": 1788170887.255636}} +{"prompt_id": 32, "output": " He spends 10*.5=<<10*.5=5>>5 hours per day\nThat means he spends 5*7=<<5*7=35>>35 hours per week\n#### 35\n\n", "correct": true, "error": null, "completion_tokens": 45, "spec_accept_length": 6.428571428571429, "spec_accept_length_from_histogram": 6.571428571428571, "spec_accept_rate": 0.37142857142857144, "spec_accepted_drafts": 39, "spec_proposed_drafts": 105, "spec_accept_histogram": [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "spec_verify_ct": 7, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "836fd5daef5144cdb439b4f103eda647", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 708, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170888.6946337, "queue_time": 0.000484466552734375, "reasoning_tokens": 0, "completion_tokens": 45, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 45}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.1361825759522617, "spec_accept_rate": 0.37142857142857144, "spec_accept_length": 6.428571428571429, "spec_num_correct_drafts": 39, "spec_num_proposed_drafts": 105, "spec_verify_ct": 7, "spec_accepted_drafts": 39, "spec_proposed_drafts": 105, "spec_correct_drafts_histogram": [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "spec_accept_histogram": [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "request_received_ts": 1788170887.2667506, "api_server_dispatch_finish_ts": 1788170887.269842, "request_finished_ts": 1788170889.4029331, "decode_throughput": 13513913.116704805, "response_sent_to_client_ts": 1788170889.4034712}} +{"prompt_id": 33, "output": " Let x be the number of silver coins Gretchen has\nGretchen has x+30 gold coins.\nx+x+30=110\n2*x=80\nx=<<40=40>>40\nGretchen has 40+30=<<40+30=70>>70 gold coins\n#### 70\n\n", "correct": true, "error": null, "completion_tokens": 68, "spec_accept_length": 4.857142857142857, "spec_accept_length_from_histogram": 4.928571428571429, "spec_accept_rate": 0.2619047619047619, "spec_accepted_drafts": 55, "spec_proposed_drafts": 210, "spec_accept_histogram": [1, 4, 1, 3, 1, 0, 1, 1, 0, 0, 0, 1, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "f2631788ee9347938c35faa4222343a1", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 695, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170890.770247, "queue_time": 0.00026726722717285156, "reasoning_tokens": 0, "completion_tokens": 68, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 68}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.734878122806549, "spec_accept_rate": 0.2619047619047619, "spec_accept_length": 4.857142857142857, "spec_num_correct_drafts": 55, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 55, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [1, 4, 1, 3, 1, 0, 1, 1, 0, 0, 0, 1, 1], "spec_accept_histogram": [1, 4, 1, 3, 1, 0, 1, 1, 0, 0, 0, 1, 1], "request_received_ts": 1788170889.4146614, "api_server_dispatch_finish_ts": 1788170889.417757, "request_finished_ts": 1788170892.1495395, "decode_throughput": 20858423.371412005, "response_sent_to_client_ts": 1788170892.150239}} +{"prompt_id": 34, "output": " Half of Raymond's jewels is 40/2 = <<40/2=20>>20\nSince Aaron has 5 more jewels than half of Raymond's jewels, he has 20+5 = <<5+20=25>>25 jewels.\nIf Siobhan has 2 fewer jewels than Aaron, he has 25-2 = 23 jewels.\n#### 23\n\n", "correct": true, "error": null, "completion_tokens": 79, "spec_accept_length": 6.583333333333333, "spec_accept_length_from_histogram": 6.666666666666667, "spec_accept_rate": 0.37777777777777777, "spec_accepted_drafts": 68, "spec_proposed_drafts": 180, "spec_accept_histogram": [0, 2, 2, 0, 2, 2, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 12, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "4ee7e8e51159464db9141628a93e943b", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 708, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170893.5660138, "queue_time": 0.00028061866760253906, "reasoning_tokens": 0, "completion_tokens": 79, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 79}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.622302720323205, "spec_accept_rate": 0.37777777777777777, "spec_accept_length": 6.583333333333333, "spec_num_correct_drafts": 68, "spec_num_proposed_drafts": 180, "spec_verify_ct": 12, "spec_accepted_drafts": 68, "spec_proposed_drafts": 180, "spec_correct_drafts_histogram": [0, 2, 2, 0, 2, 2, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 2, 2, 0, 2, 2, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788170892.1608543, "api_server_dispatch_finish_ts": 1788170892.163948, "request_finished_ts": 1788170894.783157, "decode_throughput": 18867281.430952918, "response_sent_to_client_ts": 1788170894.783741}} +{"prompt_id": 35, "output": " He scored 4*.25=<<4*.25=1>>1 point more in the second 20 minutes\nSo he scored 4+1=<<4+1=5>>5 points in the second 20 minutes\nSo in total, he scored 4+5=<<4+5=9>>9 points in the 40 minutes\n#### 9\n\n", "correct": true, "error": null, "completion_tokens": 79, "spec_accept_length": 8.777777777777779, "spec_accept_length_from_histogram": 8.88888888888889, "spec_accept_rate": 0.5259259259259259, "spec_accepted_drafts": 71, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 0, 1, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 1], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c0f2a0cb4c4241c7986ff13200f1b879", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 716, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170896.2150328, "queue_time": 0.00034117698669433594, "reasoning_tokens": 0, "completion_tokens": 79, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 79}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.348722190130502, "spec_accept_rate": 0.5259259259259259, "spec_accept_length": 8.777777777777779, "spec_num_correct_drafts": 71, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 71, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 0, 1, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 1], "request_received_ts": 1788170894.7946024, "api_server_dispatch_finish_ts": 1788170894.7978692, "request_finished_ts": 1788170897.1433246, "decode_throughput": 250.96850569911032, "response_sent_to_client_ts": 1788170897.1438336}} +{"prompt_id": 36, "output": " He eats 2 yogurts a day so over 30 days he will need 2*30 = <<2*30=60>>60 yogurts\nThe yogurt is sold as 4 units for $5.00 and he needs 60 yogurts so that’s 60/4 = <<60/4=15>>15 4-packs\nEach 4-pack is $5.00 and he needs 15 4-packs so he will spend $5*15 = $<<5*15=75.00>>75.00 on yogurt\n#### 75\n\n", "correct": true, "error": null, "completion_tokens": 124, "spec_accept_length": 5.636363636363637, "spec_accept_length_from_histogram": 5.681818181818182, "spec_accept_rate": 0.31212121212121213, "spec_accepted_drafts": 103, "spec_proposed_drafts": 330, "spec_accept_histogram": [0, 2, 3, 3, 5, 2, 1, 1, 4, 0, 0, 1], "spec_verify_ct": 22, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "60d64fa240ab46f295f1e3eb7d0f95d5", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 709, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170898.5405302, "queue_time": 0.0003113746643066406, "reasoning_tokens": 0, "completion_tokens": 124, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 124}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.6006797631271183, "spec_accept_rate": 0.31212121212121213, "spec_accept_length": 5.636363636363637, "spec_num_correct_drafts": 103, "spec_num_proposed_drafts": 330, "spec_verify_ct": 22, "spec_accepted_drafts": 103, "spec_proposed_drafts": 330, "spec_correct_drafts_histogram": [0, 2, 3, 3, 5, 2, 1, 1, 4, 0, 0, 1], "spec_accept_histogram": [0, 2, 3, 3, 5, 2, 1, 1, 4, 0, 0, 1], "request_received_ts": 1788170897.1546216, "api_server_dispatch_finish_ts": 1788170897.15787, "request_finished_ts": 1788170900.7553015, "decode_throughput": 31130287.41355333, "response_sent_to_client_ts": 1788170900.75588}} +{"prompt_id": 37, "output": " He spent $160 on video games because 8 x 20 = <<8*20=160>>160\nHe earned $165 from his sale because 160 + 5 = <<160+5=165>>165\nHe sold 11 lego sets because 11 x 15 = <<11*15=165>>165\nHe has 2 sets left because 13 - 11 = <<13-11=2>>2\n#### 2\n\n", "correct": true, "error": null, "completion_tokens": 96, "spec_accept_length": 6.857142857142857, "spec_accept_length_from_histogram": 6.928571428571429, "spec_accept_rate": 0.3952380952380952, "spec_accepted_drafts": 83, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 2, 1, 3, 2, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "23312e6485564498a0fe6e6bcc8a27a6", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170902.2157185, "queue_time": 0.0003323554992675781, "reasoning_tokens": 0, "completion_tokens": 96, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 96}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.8655119501054287, "spec_accept_rate": 0.3952380952380952, "spec_accept_length": 6.857142857142857, "spec_num_correct_drafts": 83, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 83, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 2, 1, 3, 2, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1], "spec_accept_histogram": [0, 2, 1, 3, 2, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1], "request_received_ts": 1788170900.7675862, "api_server_dispatch_finish_ts": 1788170900.7709525, "request_finished_ts": 1788170903.6330981, "decode_throughput": 27431887.395455156, "response_sent_to_client_ts": 1788170903.6335719}} +{"prompt_id": 38, "output": " He has 7-1=<<7-1=6>>6 possible running days.\nHe runs half of those so he runs 6/2=<<6/2=3>>3 days.\nSo for two days, he runs 3/2=<<3/2=1.5>>1.5 hours.\nSo in total, he runs 3+1.5+1.5=<<3+1.5+1.5=6>>6 hours.\nSo he runs 60/6=<<60/6=10>>10 mph.\n#### 10\n\n", "correct": true, "error": null, "completion_tokens": 124, "spec_accept_length": 6.526315789473684, "spec_accept_length_from_histogram": 6.578947368421052, "spec_accept_rate": 0.3719298245614035, "spec_accepted_drafts": 106, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 3, 1, 2, 2, 0, 4, 3, 1, 1, 1, 0, 0, 0, 0, 1], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "d134ebde1f024e59b90d8dce1a393197", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 711, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170905.058391, "queue_time": 0.00033736228942871094, "reasoning_tokens": 0, "completion_tokens": 124, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 124}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.3058398980647326, "spec_accept_rate": 0.3719298245614035, "spec_accept_length": 6.526315789473684, "spec_num_correct_drafts": 106, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 106, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 3, 1, 2, 2, 0, 4, 3, 1, 1, 1, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 3, 1, 2, 2, 0, 4, 3, 1, 1, 1, 0, 0, 0, 0, 1], "request_received_ts": 1788170903.6487288, "api_server_dispatch_finish_ts": 1788170903.6522913, "request_finished_ts": 1788170906.9545686, "decode_throughput": 27127502.177672796, "response_sent_to_client_ts": 1788170906.955192}} +{"prompt_id": 39, "output": " If Dana can skip at half the speed she can run, then she can run at 3*2=<<3*2=6>>6 miles per hour.\nAnd since she can run at a speed that is 4 times faster than she can walk, this means she can walk at 6/4=1.5 miles per hour.\nIf two-thirds of the time is spent walking, then she walks for 6*(2/3)=<<6*(2/3)=4>>4 hours.\nIf one-third of the time is spent running, then she runs for 6-4=<<6-4=2>>2 hours.\nThus, she runs for 2 hours at 6 miles per hour, or 2*6=<<2*6=12>>12 miles.\nShe walks for 4 hours at 1.5 miles per hour, or 4*1.5=<<4*1.5=6>>6 miles.\nThus, altogether, she travels 12+6=<<12+6=18>>18 miles.\n#### 18\n\n", "correct": true, "error": null, "completion_tokens": 225, "spec_accept_length": 7.03125, "spec_accept_length_from_histogram": 7.0625, "spec_accept_rate": 0.4041666666666667, "spec_accepted_drafts": 194, "spec_proposed_drafts": 480, "spec_accept_histogram": [0, 3, 2, 4, 6, 3, 2, 2, 2, 2, 2, 0, 1, 0, 1, 2], "spec_verify_ct": 32, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "f1860404c1d845f99c25cc042f49482d", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 740, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170908.3806145, "queue_time": 0.0004875659942626953, "reasoning_tokens": 0, "completion_tokens": 225, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 225}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.6847480637952685, "spec_accept_rate": 0.4041666666666667, "spec_accept_length": 7.03125, "spec_num_correct_drafts": 194, "spec_num_proposed_drafts": 480, "spec_verify_ct": 32, "spec_accepted_drafts": 194, "spec_proposed_drafts": 480, "spec_correct_drafts_histogram": [0, 3, 2, 4, 6, 3, 2, 2, 2, 2, 2, 0, 1, 0, 1, 2], "spec_accept_histogram": [0, 3, 2, 4, 6, 3, 2, 2, 2, 2, 2, 0, 1, 0, 1, 2], "request_received_ts": 1788170906.9674482, "api_server_dispatch_finish_ts": 1788170906.9708724, "request_finished_ts": 1788170911.6521964, "decode_throughput": 734.5027528778522, "response_sent_to_client_ts": 1788170911.6527364}} +{"prompt_id": 40, "output": " Ben’s iPhone is 1*2 = <<1*2=2>>2 years old.\nBrandon’s iPhone is 4*2 = <<4*2=8>>8 years old.\n#### 8\n\n", "correct": true, "error": null, "completion_tokens": 48, "spec_accept_length": 6.857142857142857, "spec_accept_length_from_histogram": 6.714285714285714, "spec_accept_rate": 0.38095238095238093, "spec_accepted_drafts": 40, "spec_proposed_drafts": 105, "spec_accept_histogram": [0, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "spec_verify_ct": 7, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3e4111ce49e44adabcc227370f1f9645", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 718, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170913.0856385, "queue_time": 0.0003712177276611328, "reasoning_tokens": 0, "completion_tokens": 48, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 48}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.1347756078466773, "spec_accept_rate": 0.38095238095238093, "spec_accept_length": 6.857142857142857, "spec_num_correct_drafts": 40, "spec_num_proposed_drafts": 105, "spec_verify_ct": 7, "spec_accepted_drafts": 40, "spec_proposed_drafts": 105, "spec_correct_drafts_histogram": [0, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "request_received_ts": 1788170911.6645515, "api_server_dispatch_finish_ts": 1788170911.6679451, "request_finished_ts": 1788170913.799327, "decode_throughput": 13632054.491626147, "response_sent_to_client_ts": 1788170913.7998452}} +{"prompt_id": 41, "output": " With the gemstone, Polly could throw the javelin 3 times farther than 400 feet, for a distance of 3*400=<<3*400=1200>>1200 feet.\n1200 feet is beyond the 1000-foot reach of the dragon's flames by a distance of 1200-1000=<<1200-1000=200>>200 feet.\n#### 200\n\n", "correct": true, "error": null, "completion_tokens": 86, "spec_accept_length": 6.615384615384615, "spec_accept_length_from_histogram": 6.6923076923076925, "spec_accept_rate": 0.37948717948717947, "spec_accepted_drafts": 74, "spec_proposed_drafts": 195, "spec_accept_histogram": [0, 1, 1, 2, 0, 2, 3, 0, 1, 2, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "6ff0240e27714c2887242f8eecd0c728", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 794, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170915.2833483, "queue_time": 0.00030422210693359375, "reasoning_tokens": 0, "completion_tokens": 86, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 86}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.812144835945219, "spec_accept_rate": 0.37948717948717947, "spec_accept_length": 6.615384615384615, "spec_num_correct_drafts": 74, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 74, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [0, 1, 1, 2, 0, 2, 3, 0, 1, 2, 0, 1], "spec_accept_histogram": [0, 1, 1, 2, 0, 2, 3, 0, 1, 2, 0, 1], "request_received_ts": 1788170913.811771, "api_server_dispatch_finish_ts": 1788170913.81557, "request_finished_ts": 1788170916.623916, "decode_throughput": 51259789.40746981, "response_sent_to_client_ts": 1788170916.624199}} +{"prompt_id": 42, "output": " To start the evening, there were 5 pies, each with 8 pieces, which is 5*8=<<5*8=40>>40 pieces of pie.\nIf only 14 remained, then 40-14=<<40-14=26>>26 pieces of pie had been taken by guests.\n#### 26\n\n", "correct": true, "error": null, "completion_tokens": 70, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 7.1, "spec_accept_rate": 0.4066666666666667, "spec_accepted_drafts": 61, "spec_proposed_drafts": 150, "spec_accept_histogram": [0, 0, 1, 2, 1, 0, 1, 2, 0, 2, 0, 1], "spec_verify_ct": 10, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "aa0c8ad83c654ce49999421aafd5be58", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 746, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170918.042095, "queue_time": 0.0004487037658691406, "reasoning_tokens": 0, "completion_tokens": 70, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 70}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.4132698122411966, "spec_accept_rate": 0.4066666666666667, "spec_accept_length": 7.0, "spec_num_correct_drafts": 61, "spec_num_proposed_drafts": 150, "spec_verify_ct": 10, "spec_accepted_drafts": 61, "spec_proposed_drafts": 150, "spec_correct_drafts_histogram": [0, 0, 1, 2, 1, 0, 1, 2, 0, 2, 0, 1], "spec_accept_histogram": [0, 0, 1, 2, 1, 0, 1, 2, 0, 2, 0, 1], "request_received_ts": 1788170916.634394, "api_server_dispatch_finish_ts": 1788170916.6367202, "request_finished_ts": 1788170919.0476637, "decode_throughput": 16578247.002908928, "response_sent_to_client_ts": 1788170919.048161}} +{"prompt_id": 43, "output": " If the total calorie target is 2000 and I have consumed 1800 calories then I have 2000-1800 = <<2000-1800=200>>200 calories left to eat\nIf each serving of chips has 250 calories and I only have 200 calories left to eat, then I can only eat 200/250 of a serving = 4/5 of a serving\nWe also know that a 300g bag of chips has 5 servings, hence each serving has 300g/5 = <<300/5=60>>60 grams\nIf I can only eat 4/5 of a serving, then I can eat only 60g * 4/5 = 240g/5 = 48 grams\n#### 48\n\n", "correct": true, "error": null, "completion_tokens": 161, "spec_accept_length": 6.1923076923076925, "spec_accept_length_from_histogram": 6.230769230769231, "spec_accept_rate": 0.3487179487179487, "spec_accepted_drafts": 136, "spec_proposed_drafts": 390, "spec_accept_histogram": [0, 1, 5, 3, 3, 1, 5, 3, 2, 1, 0, 1, 1], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "fbd524aa363b49a289cb675e65ac87fc", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 721, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170920.4506898, "queue_time": 0.0002930164337158203, "reasoning_tokens": 0, "completion_tokens": 161, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 161}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.021148658823222, "spec_accept_rate": 0.3487179487179487, "spec_accept_length": 6.1923076923076925, "spec_num_correct_drafts": 136, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 136, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [0, 1, 5, 3, 3, 1, 5, 3, 2, 1, 0, 1, 1], "spec_accept_histogram": [0, 1, 5, 3, 3, 1, 5, 3, 2, 1, 0, 1, 1], "request_received_ts": 1788170919.0589588, "api_server_dispatch_finish_ts": 1788170919.062263, "request_finished_ts": 1788170923.0801075, "decode_throughput": 131.50150367354524, "response_sent_to_client_ts": 1788170923.0807073}} +{"prompt_id": 44, "output": " 1 pound of beeswax and the wick make 10 candles and he wants to make 20 candles so he needs 20/10 = <<20/10=2>>2 pounds of beeswax\n1 pound of beeswax costs $10.00 and he needs 2 pounds so his supplies will cost 10*2 = $<<10*2=20.00>>20.00\nHe wants to sell each candle for $2.00 each and he's selling 20 candles so that's 2*20 = $<<2*20=40.00>>40.00\nIf he makes $40.00 selling candles and his supplies cost $20.00 then his net profit is 40-20 = $<<40-20=20.00>>20.00\n#### 20\n\n", "correct": true, "error": null, "completion_tokens": 168, "spec_accept_length": 6.222222222222222, "spec_accept_length_from_histogram": 6.333333333333333, "spec_accept_rate": 0.35555555555555557, "spec_accepted_drafts": 144, "spec_proposed_drafts": 405, "spec_accept_histogram": [1, 0, 6, 4, 1, 0, 6, 3, 1, 2, 1, 1, 1], "spec_verify_ct": 27, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "4dd241c6b9714cbc893f134ecd562359", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 737, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170924.4980018, "queue_time": 0.0002124309539794922, "reasoning_tokens": 0, "completion_tokens": 168, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 168}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.110803227871656, "spec_accept_rate": 0.35555555555555557, "spec_accept_length": 6.222222222222222, "spec_num_correct_drafts": 144, "spec_num_proposed_drafts": 405, "spec_verify_ct": 27, "spec_accepted_drafts": 144, "spec_proposed_drafts": 405, "spec_correct_drafts_histogram": [1, 0, 6, 4, 1, 0, 6, 3, 1, 2, 1, 1, 1], "spec_accept_histogram": [1, 0, 6, 4, 1, 0, 6, 3, 1, 2, 1, 1, 1], "request_received_ts": 1788170923.0916774, "api_server_dispatch_finish_ts": 1788170923.0951164, "request_finished_ts": 1788170927.2024808, "decode_throughput": 37647466.85030443, "response_sent_to_client_ts": 1788170927.203062}} +{"prompt_id": 45, "output": " If she wrote five articles on Monday, then on Tuesday she wrote 2/5* 5 = 2 more articles.\nThe total number of articles she wrote on Tuesday is 5+2 = <<5+2=7>>7\nOn Wednesday, the number of articles was twice the number she wrote on Tuesday, so it was 2*7 = <<2*7=14>>14 articles.\nOver the three days, she wrote 5+7 +14 = <<5+7+14=26>>26 articles.\nIf each article takes her 4 hours to research and write about, she spent 4*26 = <<4*26=104>>104 hours on all the articles.\n#### 104\n\n", "correct": true, "error": null, "completion_tokens": 150, "spec_accept_length": 6.25, "spec_accept_length_from_histogram": 6.291666666666667, "spec_accept_rate": 0.3527777777777778, "spec_accepted_drafts": 127, "spec_proposed_drafts": 360, "spec_accept_histogram": [0, 1, 2, 4, 5, 3, 0, 3, 3, 1, 1, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c430218e6d0b43acaa3bc6df0939f90f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 760, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170928.6509275, "queue_time": 0.00034546852111816406, "reasoning_tokens": 0, "completion_tokens": 150, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 150}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.8713552551344037, "spec_accept_rate": 0.3527777777777778, "spec_accept_length": 6.25, "spec_num_correct_drafts": 127, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 127, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [0, 1, 2, 4, 5, 3, 0, 3, 3, 1, 1, 1], "spec_accept_histogram": [0, 1, 2, 4, 5, 3, 0, 3, 3, 1, 1, 1], "request_received_ts": 1788170927.2145765, "api_server_dispatch_finish_ts": 1788170927.2181273, "request_finished_ts": 1788170931.0859318, "decode_throughput": 98.2484840542105, "response_sent_to_client_ts": 1788170931.0865207}} +{"prompt_id": 46, "output": " Let's let x be the number of post-it notes in the package that Candice purchased.\nThen, the total number of post-it notes she had when she left the store was 80+x.\nAnd if she used 220 Post-it notes at work and had 23 left, then the total number she had when she left the store was 220+23=243.\nSubtracting 80 from 243 gives 163 post-it notes.\nThe number of post-it notes in the package was 163.\n#### 163\n\n", "correct": true, "error": null, "completion_tokens": 107, "spec_accept_length": 4.458333333333333, "spec_accept_length_from_histogram": 4.5, "spec_accept_rate": 0.23333333333333334, "spec_accepted_drafts": 84, "spec_proposed_drafts": 360, "spec_accept_histogram": [1, 7, 1, 5, 1, 4, 1, 2, 2], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "73459aa2681b428fb6c2d0773635d798", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 757, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170932.5309951, "queue_time": 0.0003325939178466797, "reasoning_tokens": 0, "completion_tokens": 107, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 107}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.8755744202062488, "spec_accept_rate": 0.23333333333333334, "spec_accept_length": 4.458333333333333, "spec_num_correct_drafts": 84, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 84, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [1, 7, 1, 5, 1, 4, 1, 2, 2], "spec_accept_histogram": [1, 7, 1, 5, 1, 4, 1, 2, 2], "request_received_ts": 1788170931.0979564, "api_server_dispatch_finish_ts": 1788170931.101428, "request_finished_ts": 1788170934.9735308, "decode_throughput": 69.89575710612316, "response_sent_to_client_ts": 1788170934.9739356}} +{"prompt_id": 47, "output": " He bought 200 / 40 = <<200/40=5>>5 blue ties\nSo he bought 5 * 2 = <<5*2=10>>10 red ties\nEach red tie cost $40 * .5 = $<<40*.5=20>>20 more than blue ties\nSo they each cost $40 + $20 = $<<40+20=60>>60\nThus he spent $60 * 10 = $<<60*10=600>>600 on red ties\nSo he spent $200 + $600 = $<<200+600=800>>800 on ties\n#### 800\n\n", "correct": true, "error": null, "completion_tokens": 131, "spec_accept_length": 5.954545454545454, "spec_accept_length_from_histogram": 6.0, "spec_accept_rate": 0.3333333333333333, "spec_accepted_drafts": 110, "spec_proposed_drafts": 330, "spec_accept_histogram": [0, 0, 4, 5, 2, 2, 2, 3, 2, 1, 0, 1], "spec_verify_ct": 22, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "ae4ca2248a9246bd8c19996f1bd17e9a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 714, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170936.397959, "queue_time": 0.00032448768615722656, "reasoning_tokens": 0, "completion_tokens": 131, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 131}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.644868925213814, "spec_accept_rate": 0.3333333333333333, "spec_accept_length": 5.954545454545454, "spec_num_correct_drafts": 110, "spec_num_proposed_drafts": 330, "spec_verify_ct": 22, "spec_accepted_drafts": 110, "spec_proposed_drafts": 330, "spec_correct_drafts_histogram": [0, 0, 4, 5, 2, 2, 2, 3, 2, 1, 0, 1], "spec_accept_histogram": [0, 0, 4, 5, 2, 2, 2, 3, 2, 1, 0, 1], "request_received_ts": 1788170934.9867418, "api_server_dispatch_finish_ts": 1788170934.989878, "request_finished_ts": 1788170938.6316106, "decode_throughput": 34757578.964143425, "response_sent_to_client_ts": 1788170938.632176}} +{"prompt_id": 48, "output": " If Tracy used a piece of wire 4 feet long, since 1 foot is equal to 12 inches, the total length of the wire in inches was 12*4=<<12*4=48>>48 inches.\nSince the wire was cut into pieces 6 inches long, Tracy obtained 48/6=<<48/6=8>>8 pieces\n#### 8\n\n", "correct": true, "error": null, "completion_tokens": 81, "spec_accept_length": 6.230769230769231, "spec_accept_length_from_histogram": 6.3076923076923075, "spec_accept_rate": 0.35384615384615387, "spec_accepted_drafts": 69, "spec_proposed_drafts": 195, "spec_accept_histogram": [0, 1, 2, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "602cef132db241ce80687ec47e0ce055", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 704, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170940.0062568, "queue_time": 0.0003838539123535156, "reasoning_tokens": 0, "completion_tokens": 81, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 81}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.686933293938637, "spec_accept_rate": 0.35384615384615387, "spec_accept_length": 6.230769230769231, "spec_num_correct_drafts": 69, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 69, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [0, 1, 2, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 1], "request_received_ts": 1788170938.6451051, "api_server_dispatch_finish_ts": 1788170938.6483345, "request_finished_ts": 1788170941.3320384, "decode_throughput": 19797037.54782208, "response_sent_to_client_ts": 1788170941.332557}} +{"prompt_id": 49, "output": " The total number of units in the building will be 8 units/floor * 15 floors = <<8*15=120>>120 units.\nIf 3/4 of the building is occupied, then the total number of occupied units is 3/4 * 120 units = <<3/4*120=90>>90 units.\nThe total number of unoccupied units is 120 units - 90 units = <<120-90=30>>30 units.\n#### 30\n\n", "correct": true, "error": null, "completion_tokens": 101, "spec_accept_length": 7.214285714285714, "spec_accept_length_from_histogram": 7.285714285714286, "spec_accept_rate": 0.41904761904761906, "spec_accepted_drafts": 88, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 2, 1, 2, 1, 0, 1, 3, 0, 1, 0, 1, 0, 1, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "fc378a26e81346f682f5454ecab64b2c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 709, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170942.768243, "queue_time": 0.00035500526428222656, "reasoning_tokens": 0, "completion_tokens": 101, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 101}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.820067374035716, "spec_accept_rate": 0.41904761904761906, "spec_accept_length": 7.214285714285714, "spec_num_correct_drafts": 88, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 88, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 2, 1, 2, 1, 0, 1, 3, 0, 1, 0, 1, 0, 1, 1], "spec_accept_histogram": [0, 2, 1, 2, 1, 0, 1, 3, 0, 1, 0, 1, 0, 1, 1], "request_received_ts": 1788170941.34332, "api_server_dispatch_finish_ts": 1788170941.3466604, "request_finished_ts": 1788170944.1633873, "decode_throughput": 27419351.991828397, "response_sent_to_client_ts": 1788170944.1639774}} +{"prompt_id": 50, "output": " In a week, Lloyd's egg farm produces 252 x 7 = <<252*7=1764>>1764 eggs.\nThere are 1764/12 = <<1764/12=147>>147 dozens of eggs produced in a week.\nSo, Lloyd makes 147 x $2 = $<<147*2=294>>294 on eggs.\n#### 294\n\n", "correct": true, "error": null, "completion_tokens": 79, "spec_accept_length": 5.642857142857143, "spec_accept_length_from_histogram": 5.714285714285714, "spec_accept_rate": 0.3142857142857143, "spec_accepted_drafts": 66, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 1, 2, 2, 2, 3, 1, 0, 2, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "d9b5adec79cd4136b8cf7cfc45f3b6b5", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 702, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170945.5427258, "queue_time": 0.00029969215393066406, "reasoning_tokens": 0, "completion_tokens": 79, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 79}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.7703654430806637, "spec_accept_rate": 0.3142857142857143, "spec_accept_length": 5.642857142857143, "spec_num_correct_drafts": 66, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 66, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 1, 2, 2, 2, 3, 1, 0, 2, 0, 1], "spec_accept_histogram": [0, 1, 2, 2, 2, 3, 1, 0, 2, 0, 1], "request_received_ts": 1788170944.1768603, "api_server_dispatch_finish_ts": 1788170944.1800888, "request_finished_ts": 1788170946.9472258, "decode_throughput": 19815890.754051816, "response_sent_to_client_ts": 1788170946.9478188}} +{"prompt_id": 51, "output": " He was travelling at full speed for 4-1=<<4-1=3>>3 hours\nSo he went 3*10=<<3*10=30>>30 miles\nThat means it takes 30/6=<<30/6=5>>5 hours to get back\n#### 5\n\n", "correct": true, "error": null, "completion_tokens": 67, "spec_accept_length": 6.090909090909091, "spec_accept_length_from_histogram": 6.181818181818182, "spec_accept_rate": 0.34545454545454546, "spec_accepted_drafts": 57, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 2, 1, 2, 0, 2, 0, 0, 0, 3, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "cb56dc2d5917400cb9bfca40bf752c63", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 716, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170948.3656008, "queue_time": 0.0003681182861328125, "reasoning_tokens": 0, "completion_tokens": 67, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 67}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.5290144341997802, "spec_accept_rate": 0.34545454545454546, "spec_accept_length": 6.090909090909091, "spec_num_correct_drafts": 57, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 57, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 2, 1, 2, 0, 2, 0, 0, 0, 3, 1], "spec_accept_histogram": [0, 2, 1, 2, 0, 2, 0, 0, 0, 3, 1], "request_received_ts": 1788170946.9586985, "api_server_dispatch_finish_ts": 1788170946.9620543, "request_finished_ts": 1788170949.487713, "decode_throughput": 18206027.073603082, "response_sent_to_client_ts": 1788170949.4882185}} +{"prompt_id": 52, "output": " 30 comic books weigh 7.5 pounds because 30 x .25 = <<30*.25=7.5>>7.5\nHe needs to remove 7.5 more pounds because 15 - 7.5 = <<15-7.5=7.5>>7.5\nHe needs to remove 15 toys because 7.5 / .5 = <<15=15>>15\n#### 15\n\n", "correct": true, "error": null, "completion_tokens": 92, "spec_accept_length": 8.363636363636363, "spec_accept_length_from_histogram": 8.363636363636363, "spec_accept_rate": 0.4909090909090909, "spec_accepted_drafts": 81, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 0, 1, 0, 3, 1, 1, 2, 1, 1, 0, 0, 0, 0, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5dd6f40e1ce845c581cfd5b8b6bf1ea2", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170950.8897812, "queue_time": 0.00029587745666503906, "reasoning_tokens": 0, "completion_tokens": 92, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 92}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.497947982046753, "spec_accept_rate": 0.4909090909090909, "spec_accept_length": 8.363636363636363, "spec_num_correct_drafts": 81, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 81, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 0, 1, 0, 3, 1, 1, 2, 1, 1, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 0, 1, 0, 3, 1, 1, 2, 1, 1, 0, 0, 0, 0, 1], "request_received_ts": 1788170949.4994202, "api_server_dispatch_finish_ts": 1788170949.5027676, "request_finished_ts": 1788170951.9973683, "decode_throughput": 21503192.33802817, "response_sent_to_client_ts": 1788170951.9979472}} +{"prompt_id": 53, "output": " On Thursday the mechanic charged a total of 6*60 + 4*40 = $<<6*60+4*40=520>>520.\nOn Friday the mechanic charged a total of 12*40 = $<<12*40=480>>480.\nThus, on Thursday, the day with higher revenue, the mechanic had 520-480 = $<<520-480=40>>40 more revenue than on Friday.\n#### 40\n\n", "correct": true, "error": null, "completion_tokens": 94, "spec_accept_length": 6.714285714285714, "spec_accept_length_from_histogram": 6.785714285714286, "spec_accept_rate": 0.38571428571428573, "spec_accepted_drafts": 81, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 0, 1, 4, 0, 2, 2, 1, 2, 1, 0, 0, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "b29843a0cbe647879f8eb5b7e439cdee", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 764, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170953.4081776, "queue_time": 0.0003342628479003906, "reasoning_tokens": 0, "completion_tokens": 94, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 94}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.8082566410303116, "spec_accept_rate": 0.38571428571428573, "spec_accept_length": 6.714285714285714, "spec_num_correct_drafts": 81, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 81, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 0, 1, 4, 0, 2, 2, 1, 2, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 4, 0, 2, 2, 1, 2, 1, 0, 0, 0, 1], "request_received_ts": 1788170952.0086718, "api_server_dispatch_finish_ts": 1788170952.0121293, "request_finished_ts": 1788170954.8169284, "decode_throughput": 24775583.583178267, "response_sent_to_client_ts": 1788170954.8174517}} +{"prompt_id": 54, "output": " Patchy has just had 3 * 7 = <<3*7=21>>21 kittens\nSince Trixie's kittens are 12, both cats have 21 +12 =<<21+12=33>>33 kittens.\nCombining the adopted kittens with those from the two cats, the Doubtfire family now has 7 + 33 = <<7+33=40>>40 kittens.\n#### 40\n\n", "correct": true, "error": null, "completion_tokens": 88, "spec_accept_length": 5.176470588235294, "spec_accept_length_from_histogram": 5.235294117647059, "spec_accept_rate": 0.2823529411764706, "spec_accepted_drafts": 72, "spec_proposed_drafts": 255, "spec_accept_histogram": [0, 3, 3, 2, 3, 0, 2, 2, 0, 1, 1], "spec_verify_ct": 17, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "4ac1598caf344caf8ad8402e5268b49f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 749, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170956.2303514, "queue_time": 0.00029850006103515625, "reasoning_tokens": 0, "completion_tokens": 88, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 88}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.0979787912219763, "spec_accept_rate": 0.2823529411764706, "spec_accept_length": 5.176470588235294, "spec_num_correct_drafts": 72, "spec_num_proposed_drafts": 255, "spec_verify_ct": 17, "spec_accepted_drafts": 72, "spec_proposed_drafts": 255, "spec_correct_drafts_histogram": [0, 3, 3, 2, 3, 0, 2, 2, 0, 1, 1], "spec_accept_histogram": [0, 3, 3, 2, 3, 0, 2, 2, 0, 1, 1], "request_received_ts": 1788170954.8284464, "api_server_dispatch_finish_ts": 1788170954.8319623, "request_finished_ts": 1788170957.9264252, "decode_throughput": 20752091.233588804, "response_sent_to_client_ts": 1788170957.9269996}} +{"prompt_id": 55, "output": " Jean has 30 - 2 = <<30-2=28>>28 lollipops\nJean can fill up 28 / 2 = <<28/2=14>>14 bags with lollipops\n#### 14\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 6.125, "spec_accept_length_from_histogram": 6.25, "spec_accept_rate": 0.35, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_accept_histogram": [0, 0, 3, 2, 0, 0, 0, 0, 0, 1, 1, 1], "spec_verify_ct": 8, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e65e52bbc78a4a169fbb49666c856d8b", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 713, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170959.3429775, "queue_time": 0.00034999847412109375, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.1901370440609753, "spec_accept_rate": 0.35, "spec_accept_length": 6.125, "spec_num_correct_drafts": 42, "spec_num_proposed_drafts": 120, "spec_verify_ct": 8, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_correct_drafts_histogram": [0, 0, 3, 2, 0, 0, 0, 0, 0, 1, 1, 1], "spec_accept_histogram": [0, 0, 3, 2, 0, 0, 0, 0, 0, 1, 1, 1], "request_received_ts": 1788170957.9378848, "api_server_dispatch_finish_ts": 1788170957.9412148, "request_finished_ts": 1788170960.128022, "decode_throughput": 12630708.871951967, "response_sent_to_client_ts": 1788170960.1285446}} +{"prompt_id": 56, "output": " It costs him $14 to go to the movies because 7 + 7 = <<7+7=14>>14\nHe can go to the movies 3 times because 42 / 14 = <<42/14=3>>3\n#### 3\n\n", "correct": true, "error": null, "completion_tokens": 56, "spec_accept_length": 6.222222222222222, "spec_accept_length_from_histogram": 6.333333333333333, "spec_accept_rate": 0.35555555555555557, "spec_accepted_drafts": 48, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 1, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "8c7b7a0f725d43a3a4cf1d0eb9135bda", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 712, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170961.5381656, "queue_time": 0.0002911090850830078, "reasoning_tokens": 0, "completion_tokens": 56, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 56}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.324792387895286, "spec_accept_rate": 0.35555555555555557, "spec_accept_length": 6.222222222222222, "spec_num_correct_drafts": 48, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 48, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 1, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1], "request_received_ts": 1788170960.13944, "api_server_dispatch_finish_ts": 1788170960.1427076, "request_finished_ts": 1788170962.4642324, "decode_throughput": 14508242.309298612, "response_sent_to_client_ts": 1788170962.4648404}} +{"prompt_id": 57, "output": " The boxes can weigh up to 5000 pounds - 3755 pounds = <<5000-3755=1245>>1245 pounds in total.\nThere can be 1245 / 15 = <<1245/15=83>>83 boxes loaded onto the truck without exceeding the bridge's weight limit.\n#### 83\n\n", "correct": true, "error": null, "completion_tokens": 69, "spec_accept_length": 6.2727272727272725, "spec_accept_length_from_histogram": 6.363636363636363, "spec_accept_rate": 0.3575757575757576, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 2, 0, 0, 4, 3, 1, 0, 0, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "dbdb51eb8318431eb15249464822bc30", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 740, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170963.8661215, "queue_time": 0.0003185272216796875, "reasoning_tokens": 0, "completion_tokens": 69, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 69}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.500556231942028, "spec_accept_rate": 0.3575757575757576, "spec_accept_length": 6.2727272727272725, "spec_num_correct_drafts": 59, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 2, 0, 0, 4, 3, 1, 0, 0, 1], "spec_accept_histogram": [0, 0, 2, 0, 0, 4, 3, 1, 0, 0, 1], "request_received_ts": 1788170962.4756625, "api_server_dispatch_finish_ts": 1788170962.479111, "request_finished_ts": 1788170964.9762187, "decode_throughput": 17236648.732766762, "response_sent_to_client_ts": 1788170964.9767613}} +{"prompt_id": 58, "output": " He spent $40.00 on groceries but they charged him a 25% fee so that's 40*.25 = $10.00\nThere is also a $3.00 delivery fee and a $4.00 tip for an extra $3 + $4 = $<<3+4=7.00>>7.00 in fees\nHis groceries were $40.00, there's a $10.00 fee and another $7.00 in delivery/tip fees for a final total of $40 + $10 + $7 = $<<40+10+7=57.00>>57.00\n#### 57\n\n", "correct": true, "error": null, "completion_tokens": 132, "spec_accept_length": 5.076923076923077, "spec_accept_length_from_histogram": 5.115384615384615, "spec_accept_rate": 0.2743589743589744, "spec_accepted_drafts": 107, "spec_proposed_drafts": 390, "spec_accept_histogram": [0, 5, 5, 4, 1, 4, 1, 4, 0, 0, 1, 0, 1], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "b632f959d3c44d589277669423e2cc07", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 747, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170966.426919, "queue_time": 0.0002925395965576172, "reasoning_tokens": 0, "completion_tokens": 132, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 132}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.059209485072643, "spec_accept_rate": 0.2743589743589744, "spec_accept_length": 5.076923076923077, "spec_num_correct_drafts": 107, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 107, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [0, 5, 5, 4, 1, 4, 1, 4, 0, 0, 1, 0, 1], "spec_accept_histogram": [0, 5, 5, 4, 1, 4, 1, 4, 0, 0, 1, 0, 1], "request_received_ts": 1788170964.9874096, "api_server_dispatch_finish_ts": 1788170964.9908442, "request_finished_ts": 1788170969.0466192, "decode_throughput": 94.08200305710004, "response_sent_to_client_ts": 1788170969.046887}} +{"prompt_id": 59, "output": " First find the number of fruit in clusters: 20 fruit/cluster * 6 clusters = <<20*6=120>>120 fruit\nThen add the number of individual fruit to find the total number of fruit: 120 fruit + 67 fruit = <<120+67=187>>187 fruit\n#### 187\n\n", "correct": true, "error": null, "completion_tokens": 66, "spec_accept_length": 5.076923076923077, "spec_accept_length_from_histogram": 5.153846153846154, "spec_accept_rate": 0.27692307692307694, "spec_accepted_drafts": 54, "spec_proposed_drafts": 195, "spec_accept_histogram": [0, 2, 1, 4, 1, 3, 0, 0, 1, 0, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e47775fe69004910ae7b10397def434b", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 697, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170970.4016008, "queue_time": 0.0003666877746582031, "reasoning_tokens": 0, "completion_tokens": 66, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 66}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.659180947113782, "spec_accept_rate": 0.27692307692307694, "spec_accept_length": 5.076923076923077, "spec_num_correct_drafts": 54, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 54, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [0, 2, 1, 4, 1, 3, 0, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 2, 1, 4, 1, 3, 0, 0, 1, 0, 0, 1], "request_received_ts": 1788170969.0565703, "api_server_dispatch_finish_ts": 1788170969.0579855, "request_finished_ts": 1788170971.7157512, "decode_throughput": 331.9374122752028, "response_sent_to_client_ts": 1788170971.716194}} +{"prompt_id": 60, "output": " 20% of 25 oranges is (20/100)*25 = <<(20/100)*25=5>>5 oranges\nThe total number of bad, unripe and sour oranges is therefore 1+5+2 = <<1+5+2=8>>8 oranges\nThe rest which is 25-8 = <<25-8=17>>17 oranges are good\n#### 17\n\n", "correct": true, "error": null, "completion_tokens": 86, "spec_accept_length": 5.375, "spec_accept_length_from_histogram": 5.5, "spec_accept_rate": 0.3, "spec_accepted_drafts": 72, "spec_proposed_drafts": 240, "spec_accept_histogram": [0, 3, 2, 3, 2, 2, 0, 0, 1, 2, 0, 0, 1], "spec_verify_ct": 16, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "bbbae42ec31544da9d4ae91d37d2a5ef", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 704, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170973.1517472, "queue_time": 0.0003097057342529297, "reasoning_tokens": 0, "completion_tokens": 86, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 86}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.0415330259129405, "spec_accept_rate": 0.3, "spec_accept_length": 5.375, "spec_num_correct_drafts": 72, "spec_num_proposed_drafts": 240, "spec_verify_ct": 16, "spec_accepted_drafts": 72, "spec_proposed_drafts": 240, "spec_correct_drafts_histogram": [0, 3, 2, 3, 2, 2, 0, 0, 1, 2, 0, 0, 1], "spec_accept_histogram": [0, 3, 2, 3, 2, 2, 0, 0, 1, 2, 0, 0, 1], "request_received_ts": 1788170971.7265978, "api_server_dispatch_finish_ts": 1788170971.7308564, "request_finished_ts": 1788170974.7681308, "decode_throughput": 21622377.40819711, "response_sent_to_client_ts": 1788170974.7687805}} +{"prompt_id": 61, "output": " The total cost of the broch was 500+800=$<<500+800=1300>>1300\nThe insurance was 1300*.1=$<<1300*.1=130>>130\nSo she pays 1300+130=$<<1300+130=1430>>1430\n#### 1430\n\n", "correct": true, "error": null, "completion_tokens": 69, "spec_accept_length": 6.2727272727272725, "spec_accept_length_from_histogram": 6.363636363636363, "spec_accept_rate": 0.3575757575757576, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 1, 1, 2, 3, 2, 1, 0, 0, 0, 0, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3ed92c224f8d4df99b1781424cb302cd", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 724, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170976.1704917, "queue_time": 0.00030684471130371094, "reasoning_tokens": 0, "completion_tokens": 69, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 69}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 2.513619842939079, "spec_accept_rate": 0.3575757575757576, "spec_accept_length": 6.2727272727272725, "spec_num_correct_drafts": 59, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 1, 1, 2, 3, 2, 1, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 1, 2, 3, 2, 1, 0, 0, 0, 0, 1], "request_received_ts": 1788170974.7800016, "api_server_dispatch_finish_ts": 1788170974.7833724, "request_finished_ts": 1788170977.2936213, "decode_throughput": 14406954.228887135, "response_sent_to_client_ts": 1788170977.294123}} +{"prompt_id": 62, "output": " First find how many years Marcy works after 20 years: 30 years - 20 years = <<30-20=10>>10 years\nThen multiply that number by the amount of her pension she gets per year: 10 years * 5% = 50%\nThen multiply that percentage by the total value of the pension to find how much she gets: $50,000 * 50% = $<<50000*50*.01=25000>>25,000\n#### 25000\n\n", "correct": true, "error": null, "completion_tokens": 106, "spec_accept_length": 4.076923076923077, "spec_accept_length_from_histogram": 4.115384615384615, "spec_accept_rate": 0.2076923076923077, "spec_accepted_drafts": 81, "spec_proposed_drafts": 390, "spec_accept_histogram": [1, 4, 5, 5, 5, 5, 0, 1], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "785f7ea373de47eaadf4a082f85d3f82", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 731, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170978.7152352, "queue_time": 0.0004622936248779297, "reasoning_tokens": 0, "completion_tokens": 106, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 106}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.0503142033703625, "spec_accept_rate": 0.2076923076923077, "spec_accept_length": 4.076923076923077, "spec_num_correct_drafts": 81, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 81, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [1, 4, 5, 5, 5, 5, 0, 1], "spec_accept_histogram": [1, 4, 5, 5, 5, 5, 0, 1], "request_received_ts": 1788170977.304984, "api_server_dispatch_finish_ts": 1788170977.3082948, "request_finished_ts": 1788170981.3552983, "decode_throughput": 25227767.178339675, "response_sent_to_client_ts": 1788170981.3558595}} +{"prompt_id": 63, "output": " A year has twelve months, so a half of the year is 12/2 = <<12/2=6>>6 months,\nFor the first six months, the streaming service charged her a total of 6*140 = $<<6*140=840>>840\nFor the next half of the year, the streaming service company charged her 10/100*140 = $<<10/100*140=14>>14 less\nShe paid $140-$14 = $<<140-14=126>>126 each month for the remaining half of the year,\nIn total, she paid 126*6 = $<<126*6=756>>756 for the other half of the year for the streaming service.\nThe total amount of money she paid at the end of the year is $840+$756 = $<<840+756=1596>>1596\n#### 1596\n\n", "correct": true, "error": null, "completion_tokens": 185, "spec_accept_length": 5.606060606060606, "spec_accept_length_from_histogram": 5.636363636363637, "spec_accept_rate": 0.3090909090909091, "spec_accepted_drafts": 153, "spec_proposed_drafts": 495, "spec_accept_histogram": [0, 4, 7, 5, 0, 5, 2, 2, 5, 1, 2], "spec_verify_ct": 33, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "37ec794090594520912cfa6a92abf5d2", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 734, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788170982.8566256, "queue_time": 0.00032067298889160156, "reasoning_tokens": 0, "completion_tokens": 185, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 185}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.821898098103702, "spec_accept_rate": 0.3090909090909091, "spec_accept_length": 5.606060606060606, "spec_num_correct_drafts": 153, "spec_num_proposed_drafts": 495, "spec_verify_ct": 33, "spec_accepted_drafts": 153, "spec_proposed_drafts": 495, "spec_correct_drafts_histogram": [0, 4, 7, 5, 0, 5, 2, 2, 5, 1, 2], "spec_accept_histogram": [0, 4, 7, 5, 0, 5, 2, 2, 5, 1, 2], "request_received_ts": 1788170981.3679082, "api_server_dispatch_finish_ts": 1788170981.3712585, "request_finished_ts": 1788170986.1898065, "decode_throughput": 140.27188270750304, "response_sent_to_client_ts": 1788170986.1903255}} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.log new file mode 100644 index 0000000..84fdaea --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c1.log @@ -0,0 +1,28 @@ +gsm8k_c1 174.1.60.1: docker run --rm --network host -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/../../../datasets/gsm8k/test.jsonl:/dataset/gsm8k.jsonl:ro -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/bench_gsm8k_acceptance.py:/bench.py:ro -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740:/results --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 /bench.py --data-path /dataset/gsm8k.jsonl --num-questions 64 --num-shots 5 --max-new-tokens 512 --temperature 0 --top-p 1 --parallel 1 --speculative-algorithm DFLASH --host 174.1.60.1 --port 31000 --backend srt --output-file /results/bench/gsm8k_c1.jsonl --summary-file /results/bench/gsm8k_c1_summary.json +/opt/kimi-dflash/python/sglang/srt/layers/quantization/awq/awq.py:53: UserWarning: Only CUDA, HIP and XPU support AWQ currently. + warnings.warn(f"Only CUDA, HIP and XPU support AWQ currently.") +/opt/kimi-dflash/python/sglang/srt/layers/quantization/gguf.py:69: UserWarning: Only CUDA, MUSA and NPU support GGUF quantization currently. + warnings.warn(f"Only CUDA, MUSA and NPU support GGUF quantization currently.") + 0%| | 0/64 [00:00>9 duck eggs a day.\nShe makes 9 * 2 = $<<9*2=18>>18 every day at the farmer’s market.\n#### 18\n\n", "correct": true, "error": null, "completion_tokens": 56, "spec_accept_length": 14.0, "spec_accept_length_from_histogram": 16.0, "spec_accept_rate": 1.0, "spec_accepted_drafts": 60, "spec_proposed_drafts": 60, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "spec_verify_ct": 4, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "848073136d5b44ae9c495cd42713e46a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 730, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171023.1028814, "queue_time": 0.0005366802215576172, "reasoning_tokens": 0, "completion_tokens": 56, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 56}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.1312459097243845, "spec_accept_rate": 1.0, "spec_accept_length": 14.0, "spec_num_correct_drafts": 60, "spec_num_proposed_drafts": 60, "spec_verify_ct": 4, "spec_accepted_drafts": 60, "spec_proposed_drafts": 60, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "request_received_ts": 1788171020.7597132, "api_server_dispatch_finish_ts": 1788171020.76377, "request_finished_ts": 1788171024.8909593, "decode_throughput": 11111157.162746944, "response_sent_to_client_ts": 1788171024.8915987}} +{"prompt_id": 1, "output": " It takes 2/2=<<2/2=1>>1 bolt of white fiber\nSo the total amount of fabric is 2+1=<<2+1=3>>3 bolts of fabric\n#### 3\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 16.333333333333332, "spec_accept_length_from_histogram": 16.0, "spec_accept_rate": 1.0, "spec_accepted_drafts": 45, "spec_proposed_drafts": 45, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], "spec_verify_ct": 3, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "0acc787b45ac41ecaddf8af7ce2b017a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 694, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171023.33857, "queue_time": 0.00016498565673828125, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.07897964399308, "spec_accept_rate": 1.0, "spec_accept_length": 16.333333333333332, "spec_num_correct_drafts": 45, "spec_num_proposed_drafts": 45, "spec_verify_ct": 3, "spec_accepted_drafts": 45, "spec_proposed_drafts": 45, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], "request_received_ts": 1788171020.766183, "api_server_dispatch_finish_ts": 1788171020.7694356, "request_finished_ts": 1788171025.8451626, "decode_throughput": 9327591.630078726, "response_sent_to_client_ts": 1788171025.8457088}} +{"prompt_id": 2, "output": " The cost of the house and repairs came out to 80,000+50,000=$<<80000+50000=130000>>130,000\nHe increased the value of the house by 80,000*1.5=<<80000*1.5=120000>>120,000\nSo the new value of the house is 120,000+80,000=$<<120000+80000=200000>>200,000\nSo he made a profit of 200,000-130,000=$<<200000-130000=70000>>70,000\n#### 70000\n\n", "correct": true, "error": null, "completion_tokens": 131, "spec_accept_length": 14.555555555555555, "spec_accept_length_from_histogram": 14.666666666666666, "spec_accept_rate": 0.9111111111111111, "spec_accepted_drafts": 123, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e985a8f76559406e80304386ce2a7a0f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 717, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171024.3587296, "queue_time": 0.0016186237335205078, "reasoning_tokens": 0, "completion_tokens": 131, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 131}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.843204802367836, "spec_accept_rate": 0.9111111111111111, "spec_accept_length": 14.555555555555555, "spec_num_correct_drafts": 123, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 123, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7], "request_received_ts": 1788171020.7704744, "api_server_dispatch_finish_ts": 1788171020.7735264, "request_finished_ts": 1788171027.6136792, "decode_throughput": 34461532.43303296, "response_sent_to_client_ts": 1788171027.6141963}} +{"prompt_id": 3, "output": " He sprints 3*3=<<3*3=9>>9 times\nSo he runs 9*60=<<9*60=540>>540 meters\n#### 540\n\n", "correct": true, "error": null, "completion_tokens": 41, "spec_accept_length": 13.666666666666666, "spec_accept_length_from_histogram": 14.0, "spec_accept_rate": 0.8666666666666667, "spec_accepted_drafts": 39, "spec_proposed_drafts": 45, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2], "spec_verify_ct": 3, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "f926e87789ac4c85ae7262439c13f475", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 702, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171024.3587296, "queue_time": 0.0013353824615478516, "reasoning_tokens": 0, "completion_tokens": 41, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 41}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.413379465695471, "spec_accept_rate": 0.8666666666666667, "spec_accept_length": 13.666666666666666, "spec_num_correct_drafts": 39, "spec_num_proposed_drafts": 45, "spec_verify_ct": 3, "spec_accepted_drafts": 39, "spec_proposed_drafts": 45, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2], "request_received_ts": 1788171020.7783697, "api_server_dispatch_finish_ts": 1788171020.7815278, "request_finished_ts": 1788171026.191749, "decode_throughput": 12354285.332949806, "response_sent_to_client_ts": 1788171026.1922545}} +{"prompt_id": 4, "output": " If each chicken eats 3 cups of feed per day, then for 20 chickens they would need 3*20=<<3*20=60>>60 cups of feed per day.\nIf she feeds the flock 15 cups of feed in the morning, and 25 cups in the afternoon, then the final meal would require 60-15-25=<<60-15-25=20>>20 cups of chicken feed.\n#### 20\n\n", "correct": true, "error": null, "completion_tokens": 94, "spec_accept_length": 15.666666666666666, "spec_accept_length_from_histogram": 16.0, "spec_accept_rate": 1.0, "spec_accepted_drafts": 90, "spec_proposed_drafts": 90, "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "spec_verify_ct": 6, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "f07da8d9d24a4aabb25d0f8b0fdf4990", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 775, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171024.3587294, "queue_time": 0.0011067390441894531, "reasoning_tokens": 0, "completion_tokens": 94, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 94}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.137197333853692, "spec_accept_rate": 1.0, "spec_accept_length": 15.666666666666666, "spec_num_correct_drafts": 90, "spec_num_proposed_drafts": 90, "spec_verify_ct": 6, "spec_accepted_drafts": 90, "spec_proposed_drafts": 90, "spec_correct_drafts_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "spec_accept_histogram": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "request_received_ts": 1788171020.783438, "api_server_dispatch_finish_ts": 1788171020.7867067, "request_finished_ts": 1788171026.9206352, "decode_throughput": 23366792.94068094, "response_sent_to_client_ts": 1788171026.9211564}} +{"prompt_id": 5, "output": " The discount price of one glass is 60/100 * 5 = $<<60/100*5=3>>3.\nIf every second glass is cheaper, that means Kylar is going to buy 16 / 2 = <<16/2=8>>8 cheaper glasses.\nSo for the cheaper glasses, Kylar is going to pay 8 * 3 = $<<8*3=24>>24.\nAnd for the regular-priced glasses, Kylar will pay 8 * 5 = $<<8*5=40>>40.\nSo in total Kylar needs to pay 24 + 40 = $<<24+40=64>>64 for the glasses he wants to buy.\n#### 64\n\n", "correct": true, "error": null, "completion_tokens": 150, "spec_accept_length": 7.142857142857143, "spec_accept_length_from_histogram": 7.190476190476191, "spec_accept_rate": 0.4126984126984127, "spec_accepted_drafts": 130, "spec_proposed_drafts": 315, "spec_accept_histogram": [0, 0, 0, 6, 4, 1, 2, 2, 0, 2, 1, 0, 2, 1], "spec_verify_ct": 21, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "09598fb30f674038b87fe902d127895a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 720, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171024.3587298, "queue_time": 0.0008935928344726562, "reasoning_tokens": 0, "completion_tokens": 150, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 150}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 9.352511872071773, "spec_accept_rate": 0.4126984126984127, "spec_accept_length": 7.142857142857143, "spec_num_correct_drafts": 130, "spec_num_proposed_drafts": 315, "spec_verify_ct": 21, "spec_accepted_drafts": 130, "spec_proposed_drafts": 315, "spec_correct_drafts_histogram": [0, 0, 0, 6, 4, 1, 2, 2, 0, 2, 1, 0, 2, 1], "spec_accept_histogram": [0, 0, 0, 6, 4, 1, 2, 2, 0, 2, 1, 0, 2, 1], "request_received_ts": 1788171020.7881374, "api_server_dispatch_finish_ts": 1788171020.7912104, "request_finished_ts": 1788171030.1406493, "decode_throughput": 45986643.22391492, "response_sent_to_client_ts": 1788171030.1411421}} +{"prompt_id": 6, "output": " If Seattle has 20 sheep, Charleston has 4 * 20 sheep = <<20*4=80>>80 sheep\nToulouse has twice as many sheep as Charleston, which is 2 * 80 sheep = <<2*80=160>>160 sheep\nTogether, the three has 20 sheep + 160 sheep + 80 sheep = <<20+160+80=260>>260 sheep\n#### 260\n\n", "correct": true, "error": null, "completion_tokens": 90, "spec_accept_length": 6.428571428571429, "spec_accept_length_from_histogram": 6.428571428571429, "spec_accept_rate": 0.3619047619047619, "spec_accepted_drafts": 76, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 2, 3, 2, 0, 0, 2, 2, 0, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "0d1651250fe3410ba33665048ed44256", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 709, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171024.3587296, "queue_time": 0.0006866455078125, "reasoning_tokens": 0, "completion_tokens": 90, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 90}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.772955406922847, "spec_accept_rate": 0.3619047619047619, "spec_accept_length": 6.428571428571429, "spec_num_correct_drafts": 76, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 76, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 2, 3, 2, 0, 0, 2, 2, 0, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 2, 3, 2, 0, 0, 2, 2, 0, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788171020.7930553, "api_server_dispatch_finish_ts": 1788171020.79599, "request_finished_ts": 1788171028.5660107, "decode_throughput": 27233691.17583357, "response_sent_to_client_ts": 1788171028.5665023}} +{"prompt_id": 7, "output": " First find how many gigabytes are in 40% of the file: 200 GB * 40% = <<200*40*.01=80>>80 GB\nThen divide that number by the download rate to find the time until Windows restarts: 80 GB / 2 GB/minute = <<80/2=40>>40 minutes\nThen find the time to download the whole file after the restart: 200 GB / 2 GB/minute = <<200/2=100>>100 minutes\nThen add the time to download 40% of the file, to download the whole file, and to wait for Windows to update: 40 minutes + 100 minutes + 20 minutes = <<40+100+20=160>>160 minutes\n#### 160\n\n", "correct": true, "error": null, "completion_tokens": 161, "spec_accept_length": 6.44, "spec_accept_length_from_histogram": 6.44, "spec_accept_rate": 0.3626666666666667, "spec_accepted_drafts": 136, "spec_proposed_drafts": 375, "spec_accept_histogram": [0, 1, 6, 6, 0, 3, 1, 1, 2, 0, 1, 1, 1, 1, 0, 1], "spec_verify_ct": 25, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "ff09165a10204fd3aa5ec5f4ab8eaa54", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 733, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171024.3587298, "queue_time": 0.0004792213439941406, "reasoning_tokens": 0, "completion_tokens": 161, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 161}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 10.349999848753214, "spec_accept_rate": 0.3626666666666667, "spec_accept_length": 6.44, "spec_num_correct_drafts": 136, "spec_num_proposed_drafts": 375, "spec_verify_ct": 25, "spec_accepted_drafts": 136, "spec_proposed_drafts": 375, "spec_correct_drafts_histogram": [0, 1, 6, 6, 0, 3, 1, 1, 2, 0, 1, 1, 1, 1, 0, 1], "spec_accept_histogram": [0, 1, 6, 6, 0, 3, 1, 1, 2, 0, 1, 1, 1, 1, 0, 1], "request_received_ts": 1788171020.7974195, "api_server_dispatch_finish_ts": 1788171020.8003094, "request_finished_ts": 1788171031.1474192, "decode_throughput": 41886795.52358893, "response_sent_to_client_ts": 1788171031.1479979}} +{"prompt_id": 8, "output": " When he turned around he was 3*60=<<3*60=180>>180 miles from home\nHe was only able to drive 4-2=<<4-2=2>>2 hours in the first four hours\nIn half an hour he goes 30*.5=<<30*.5=15>>15 miles\nHe then drives another 2-.5=<<2-.5=1.5>>1.5 hours\nIn that time he goes 80*1.5=<<80*1.5=120>>120 miles\nSo he drove 120+15=<<120+15=135>>135 miles\nSo he is 180-135=<<180-135=45>>45 miles away from home\n#### 45\n\n", "correct": true, "error": null, "completion_tokens": 161, "spec_accept_length": 6.708333333333333, "spec_accept_length_from_histogram": 6.75, "spec_accept_rate": 0.38333333333333336, "spec_accepted_drafts": 138, "spec_proposed_drafts": 360, "spec_accept_histogram": [0, 1, 3, 3, 4, 3, 0, 4, 1, 2, 1, 0, 1, 0, 0, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "18a9b297329c48818d49be7b1f96c1a8", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 769, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171027.1447597, "queue_time": 0.0002117156982421875, "reasoning_tokens": 0, "completion_tokens": 161, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 161}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.962136375717819, "spec_accept_rate": 0.38333333333333336, "spec_accept_length": 6.708333333333333, "spec_num_correct_drafts": 138, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 138, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [0, 1, 3, 3, 4, 3, 0, 4, 1, 2, 1, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 1, 3, 3, 4, 3, 0, 4, 1, 2, 1, 0, 1, 0, 0, 1], "request_received_ts": 1788171024.9036212, "api_server_dispatch_finish_ts": 1788171024.907199, "request_finished_ts": 1788171032.8657577, "decode_throughput": 43241553.44575887, "response_sent_to_client_ts": 1788171032.8662865}} +{"prompt_id": 9, "output": " Eliza is entitled to 45 -40 = <<45-40=5>>5 hours overtime pay.\nHer hourly rate for the overtime pay is $10 x 1.2 = $<<10*1.2=12>>12.\nSo, Eliza will receive $12 x 5 =$<<12*5=60>>60 for overtime pay.\nHer regular weekly earning is $10 x 40 = $<<10*40=400>>400.\nThus, Eliza will receive a total of $400 + $60 = $<<400+60=460>>460 for this week's work.\n#### 460\n\n", "correct": true, "error": null, "completion_tokens": 130, "spec_accept_length": 5.416666666666667, "spec_accept_length_from_histogram": 5.458333333333333, "spec_accept_rate": 0.2972222222222222, "spec_accepted_drafts": 107, "spec_proposed_drafts": 360, "spec_accept_histogram": [0, 2, 4, 5, 2, 5, 1, 0, 3, 1, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c6e8f479e25a40bf82a7559d898a4e36", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 725, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171028.3827302, "queue_time": 0.00017261505126953125, "reasoning_tokens": 0, "completion_tokens": 130, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 130}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 9.063318707048893, "spec_accept_rate": 0.2972222222222222, "spec_accept_length": 5.416666666666667, "spec_num_correct_drafts": 107, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 107, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [0, 2, 4, 5, 2, 5, 1, 0, 3, 1, 1], "spec_accept_histogram": [0, 2, 4, 5, 2, 5, 1, 0, 3, 1, 1], "request_received_ts": 1788171025.8568895, "api_server_dispatch_finish_ts": 1788171025.860344, "request_finished_ts": 1788171034.9202082, "decode_throughput": 35953976.715379626, "response_sent_to_client_ts": 1788171034.920715}} +{"prompt_id": 10, "output": " The number of downloads of the program in the second month increased to 3*60 = <<3*60=180>>180\nIn the first two months, the total number of downloads of the program was 180+60 = <<180+60=240>>240\nIn the third month, the number of downloads of the program reduced by 30/100*180 = <<30/100*180=54>>54\nThere were 180-54 = <<180-54=126>>126 downloads in the third month.\nIn the three months, the total number of downloads of the program was 126+240 = <<126+240=366>>366\n#### 366\n\n", "correct": true, "error": null, "completion_tokens": 142, "spec_accept_length": 6.761904761904762, "spec_accept_length_from_histogram": 6.714285714285714, "spec_accept_rate": 0.38095238095238093, "spec_accepted_drafts": 120, "spec_proposed_drafts": 315, "spec_accept_histogram": [0, 0, 0, 6, 1, 2, 6, 1, 2, 1, 2], "spec_verify_ct": 21, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "29ab70fb26a148be92b65b0093e4e998", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 726, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171028.793264, "queue_time": 0.00014519691467285156, "reasoning_tokens": 0, "completion_tokens": 142, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 142}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 8.149700137786567, "spec_accept_rate": 0.38095238095238093, "spec_accept_length": 6.761904761904762, "spec_num_correct_drafts": 120, "spec_num_proposed_drafts": 315, "spec_verify_ct": 21, "spec_accepted_drafts": 120, "spec_proposed_drafts": 315, "spec_correct_drafts_histogram": [0, 0, 0, 6, 1, 2, 6, 1, 2, 1, 2], "spec_accept_histogram": [0, 0, 0, 6, 1, 2, 6, 1, 2, 1, 2], "request_received_ts": 1788171026.2023664, "api_server_dispatch_finish_ts": 1788171026.2057364, "request_finished_ts": 1788171034.3520665, "decode_throughput": 45836390.30699364, "response_sent_to_client_ts": 1788171034.3525615}} +{"prompt_id": 11, "output": " The total charge for the doughnuts was 3 x $68 = $<<3*68=204>>204.\nThe total charge for the mini cupcakes was 2 x $80 = $<<2*80=160>>160.\nThe total charge for the mini cheesecakes was 6 x $55 = $<<6*55=330>>330.\nTherefore the total amount Toula paid for the pastries was $204 + $160 + $330 = $<<204+160+330=694>>694.\n#### 694\n\n", "correct": true, "error": null, "completion_tokens": 111, "spec_accept_length": 7.928571428571429, "spec_accept_length_from_histogram": 8.0, "spec_accept_rate": 0.4666666666666667, "spec_accepted_drafts": 98, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 3, 1, 1, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 0, 2], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3f9fef92411447ca968d88d0fb4b7582", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 727, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171029.9221838, "queue_time": 0.00012731552124023438, "reasoning_tokens": 0, "completion_tokens": 111, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 111}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.115966156125069, "spec_accept_rate": 0.4666666666666667, "spec_accept_length": 7.928571428571429, "spec_num_correct_drafts": 98, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 98, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 3, 1, 1, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 0, 2], "spec_accept_histogram": [0, 3, 1, 1, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 0, 2], "request_received_ts": 1788171026.9315464, "api_server_dispatch_finish_ts": 1788171026.9348552, "request_finished_ts": 1788171034.0475128, "decode_throughput": 408.72742744392724, "response_sent_to_client_ts": 1788171034.0480049}} +{"prompt_id": 12, "output": " He makes $10.5 selling lemons each year because 7 x 1.5 = <<7*1.5=10.5>>10.5\nHe earns $7.5 each year from the lemon tree because 10.5 - 3 = <<10.5-3=7.5>>7.5\nIt will take 12 years to earn enough to pay off the tree because 90 / 7.5 = <<90/7.5=12>>12\nHe will make money in year 13 because 12 + 1 = <<12+1=13>>13\n#### 13\n\n", "correct": true, "error": null, "completion_tokens": 133, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 6.947368421052632, "spec_accept_rate": 0.39649122807017545, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 0, 4, 2, 3, 1, 3, 0, 0, 3, 1, 0, 1, 0, 0, 1], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e4e838a9eacc4db1b9c8641db898300a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 733, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171030.3793256, "queue_time": 0.00014209747314453125, "reasoning_tokens": 0, "completion_tokens": 133, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 133}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.992074851877987, "spec_accept_rate": 0.39649122807017545, "spec_accept_length": 7.0, "spec_num_correct_drafts": 113, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 0, 4, 2, 3, 1, 3, 0, 0, 3, 1, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 0, 4, 2, 3, 1, 3, 0, 0, 3, 1, 0, 1, 0, 0, 1], "request_received_ts": 1788171027.6220148, "api_server_dispatch_finish_ts": 1788171027.6253722, "request_finished_ts": 1788171035.6140895, "decode_throughput": 43941689.89862037, "response_sent_to_client_ts": 1788171035.6145766}} +{"prompt_id": 13, "output": " First multiply the five remaining vacuum cleaners by two to find out how many Melanie had before she visited the orange house: 5 * 2 = <<5*2=10>>10\nThen add two to figure out how many vacuum cleaners she had before visiting the red house: 10 + 2 = <<10+2=12>>12\nNow we know that 2/3 * x = 12, where x is the number of vacuum cleaners Melanie started with. We can find x by dividing each side of the equation by 2/3, which produces x = 18\n#### 18\n\n", "correct": true, "error": null, "completion_tokens": 126, "spec_accept_length": 5.478260869565218, "spec_accept_length_from_histogram": 5.521739130434782, "spec_accept_rate": 0.30144927536231886, "spec_accepted_drafts": 104, "spec_proposed_drafts": 345, "spec_accept_histogram": [0, 2, 5, 4, 1, 3, 1, 3, 3, 0, 1], "spec_verify_ct": 23, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "b73f38c3bc0f4b8db6f3a49334b359e8", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 725, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171030.8956745, "queue_time": 0.000148773193359375, "reasoning_tokens": 0, "completion_tokens": 126, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 126}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 9.123382485937327, "spec_accept_rate": 0.30144927536231886, "spec_accept_length": 5.478260869565218, "spec_num_correct_drafts": 104, "spec_num_proposed_drafts": 345, "spec_verify_ct": 23, "spec_accepted_drafts": 104, "spec_proposed_drafts": 345, "spec_correct_drafts_histogram": [0, 2, 5, 4, 1, 3, 1, 3, 3, 0, 1], "spec_accept_histogram": [0, 2, 5, 4, 1, 3, 1, 3, 3, 0, 1], "request_received_ts": 1788171028.57671, "api_server_dispatch_finish_ts": 1788171028.5799716, "request_finished_ts": 1788171037.7000926, "decode_throughput": 27186090.338262103, "response_sent_to_client_ts": 1788171037.7010555}} +{"prompt_id": 14, "output": " There are 20 x 20/100 = <<20*20/100=4>>4 students who enrolled in contemporary dance.\nSo, 20 - 4 = <<20-4=16>>16 students are enrolled in either jazz or hip-hop dance.\nThere are 16 x 25/100 = <<16*25/100=4>>4 students who enrolled in jazz dance.\nHence, 16 - 4 = <<16-4=12>>12 students enrolled in hip-hop dance.\nThis is 12/20 x 100% = 60% of the entire students.\n#### 60\n\n", "correct": true, "error": null, "completion_tokens": 128, "spec_accept_length": 6.7368421052631575, "spec_accept_length_from_histogram": 6.7894736842105265, "spec_accept_rate": 0.38596491228070173, "spec_accepted_drafts": 110, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 1, 4, 1, 1, 4, 1, 1, 1, 2, 0, 1, 2], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5bc5c6ed56d542deb92f92da62b5d36f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 717, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171031.9601445, "queue_time": 0.00015735626220703125, "reasoning_tokens": 0, "completion_tokens": 128, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 128}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.549075374845415, "spec_accept_rate": 0.38596491228070173, "spec_accept_length": 6.7368421052631575, "spec_num_correct_drafts": 110, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 110, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 1, 4, 1, 1, 4, 1, 1, 1, 2, 0, 1, 2], "spec_accept_histogram": [0, 1, 4, 1, 1, 4, 1, 1, 1, 2, 0, 1, 2], "request_received_ts": 1788171030.1515276, "api_server_dispatch_finish_ts": 1788171030.1548295, "request_finished_ts": 1788171037.700603, "decode_throughput": 61661863.73411711, "response_sent_to_client_ts": 1788171037.7021062}} +{"prompt_id": 15, "output": " If he purchases jewelry, he will make a profit of 2.5% which is $5000*(2.5/100) = $<<5000*(2.5/100)=125>>125\nIf he purchases electronic gadgets, he will make a profit of 1.2% which is $8000*(1.2/100) = $<<8000*(1.2/100)=96>>96\nIf he wants to maximize profit, since $125 > $96, he will choose to purchase jewelry, thereby making a profit of $<<125=125>>125\n#### 125\n\n", "correct": true, "error": null, "completion_tokens": 128, "spec_accept_length": 7.529411764705882, "spec_accept_length_from_histogram": 7.588235294117647, "spec_accept_rate": 0.4392156862745098, "spec_accepted_drafts": 112, "spec_proposed_drafts": 255, "spec_accept_histogram": [0, 1, 0, 4, 3, 1, 1, 2, 1, 0, 1, 0, 0, 0, 1, 2], "spec_verify_ct": 17, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "a857109e76af45a484d7f3f1ca46860c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 758, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171033.1573238, "queue_time": 0.00012373924255371094, "reasoning_tokens": 0, "completion_tokens": 128, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 128}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.794642649590969, "spec_accept_rate": 0.4392156862745098, "spec_accept_length": 7.529411764705882, "spec_num_correct_drafts": 112, "spec_num_proposed_drafts": 255, "spec_verify_ct": 17, "spec_accepted_drafts": 112, "spec_proposed_drafts": 255, "spec_correct_drafts_histogram": [0, 1, 0, 4, 3, 1, 1, 2, 1, 0, 1, 0, 0, 0, 1, 2], "spec_accept_histogram": [0, 1, 0, 4, 3, 1, 1, 2, 1, 0, 1, 0, 0, 0, 1, 2], "request_received_ts": 1788171031.1587017, "api_server_dispatch_finish_ts": 1788171031.162011, "request_finished_ts": 1788171037.9533443, "decode_throughput": 27300342.672272272, "response_sent_to_client_ts": 1788171037.9538622}} +{"prompt_id": 16, "output": " On the first day, the trains covered 2 trains * 80 miles/train = <<2*80=160>>160 miles together.\nThey also covered 150 miles/train * 2 trains = <<150*2=300>>300 miles together on the second day.\nThe combined distance the two trains covered in the two days is 300 miles + 160 miles = <<300+160=460>>460 miles\nThe average distance for the two days is 460 miles / 2 trains = <<460/2=230>>230 miles/train\n#### 230\n\n", "correct": true, "error": null, "completion_tokens": 117, "spec_accept_length": 6.157894736842105, "spec_accept_length_from_histogram": 6.2105263157894735, "spec_accept_rate": 0.3473684210526316, "spec_accepted_drafts": 99, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 0, 2, 7, 2, 1, 0, 3, 0, 1, 2, 1], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "415d10be71f548c8b034d08878a4aca3", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 717, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171034.622519, "queue_time": 0.00016617774963378906, "reasoning_tokens": 0, "completion_tokens": 117, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 117}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.528884414117783, "spec_accept_rate": 0.3473684210526316, "spec_accept_length": 6.157894736842105, "spec_num_correct_drafts": 99, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 99, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 0, 2, 7, 2, 1, 0, 3, 0, 1, 2, 1], "spec_accept_histogram": [0, 0, 2, 7, 2, 1, 0, 3, 0, 1, 2, 1], "request_received_ts": 1788171032.8771527, "api_server_dispatch_finish_ts": 1788171032.8804798, "request_finished_ts": 1788171039.406037, "decode_throughput": 24764698.595088974, "response_sent_to_client_ts": 1788171039.40656}} +{"prompt_id": 17, "output": " First find the total amount Jill makes per week teaching: $20/hour * 35 hours/week = $<<20*35=700>>700/week\nThen find the total amount Jill makes per week coaching: $30/hour * 15 hours/week = $<<30*15=450>>450/week\nThen add those two amounts to find the total amount Jill makes per week: $700/week + $450/week = $<<700+450=1150>>1150/week\nThen multiply that number by the number of weeks Jill works in a year to find her annual salary: $1150/week * 50 weeks/year = $<<1150*50=57500>>57,500\n#### 57500\n\n", "correct": true, "error": null, "completion_tokens": 150, "spec_accept_length": 6.521739130434782, "spec_accept_length_from_histogram": 6.608695652173913, "spec_accept_rate": 0.3739130434782609, "spec_accepted_drafts": 129, "spec_proposed_drafts": 345, "spec_accept_histogram": [0, 2, 5, 3, 0, 4, 2, 2, 1, 0, 0, 0, 1, 1, 1, 1], "spec_verify_ct": 23, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e81f0f7b73aa4b7a82980ac581bcfd4b", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 719, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171036.3461308, "queue_time": 0.0004336833953857422, "reasoning_tokens": 0, "completion_tokens": 150, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 150}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.630079943686724, "spec_accept_rate": 0.3739130434782609, "spec_accept_length": 6.521739130434782, "spec_num_correct_drafts": 129, "spec_num_proposed_drafts": 345, "spec_verify_ct": 23, "spec_accepted_drafts": 129, "spec_proposed_drafts": 345, "spec_correct_drafts_histogram": [0, 2, 5, 3, 0, 4, 2, 2, 1, 0, 0, 0, 1, 1, 1, 1], "spec_accept_histogram": [0, 2, 5, 3, 0, 4, 2, 2, 1, 0, 0, 0, 1, 1, 1, 1], "request_received_ts": 1788171034.0583067, "api_server_dispatch_finish_ts": 1788171034.0615325, "request_finished_ts": 1788171041.6883867, "decode_throughput": 52472132.42899311, "response_sent_to_client_ts": 1788171041.6888897}} +{"prompt_id": 18, "output": " She eats 3 eggs every day and there are 7 days in a week so she eats 3*7 = <<3*7=21>>21 eggs a week\nAfter 4 weeks she will have eaten 4*21 = <<4*21=84>>84 eggs\nThere are 12 eggs in 1 dozen and she'll eat 84 eggs so that's 84/12 = <<84/12=7>>7 dozen eggs\n#### 7\n\n", "correct": true, "error": null, "completion_tokens": 98, "spec_accept_length": 5.764705882352941, "spec_accept_length_from_histogram": 5.823529411764706, "spec_accept_rate": 0.3215686274509804, "spec_accepted_drafts": 82, "spec_proposed_drafts": 255, "spec_accept_histogram": [0, 1, 4, 3, 1, 4, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1], "spec_verify_ct": 17, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "1f8113c421dc4ba0bffcd2ce18988f86", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 695, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171037.4735415, "queue_time": 0.00016260147094726562, "reasoning_tokens": 0, "completion_tokens": 98, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 98}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.947669697925448, "spec_accept_rate": 0.3215686274509804, "spec_accept_length": 5.764705882352941, "spec_num_correct_drafts": 82, "spec_num_proposed_drafts": 255, "spec_verify_ct": 17, "spec_accepted_drafts": 82, "spec_proposed_drafts": 255, "spec_correct_drafts_histogram": [0, 1, 4, 3, 1, 4, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 4, 3, 1, 4, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1], "request_received_ts": 1788171034.3628626, "api_server_dispatch_finish_ts": 1788171034.366082, "request_finished_ts": 1788171041.3105323, "decode_throughput": 27247339.941922825, "response_sent_to_client_ts": 1788171041.311102}} +{"prompt_id": 19, "output": " First figure out how many hours it takes to hike a 12-mile trail at 4 mph by dividing the distance by the speed: 12 miles / 4 mph = <<12/4=3>>3 hours\nNext subtract the time Marissa already spent walking to find out how much time she has left: 3 hours - 1 hour - 1 hour = <<3-1-1=1>>1 hour\nNow figure out how much distance she has left by subtracting the distance she already traveled from the total distance: 12 miles - 4 miles - 2 miles = <<12-4-2=6>>6 miles\nNow divide the remaining distance by the remaining time to find out how fast in miles per hour Marissa has to travel: 6 miles / 1 hour = <<6/1=6>>6 mph\n#### 6\n\n", "correct": true, "error": null, "completion_tokens": 181, "spec_accept_length": 6.241379310344827, "spec_accept_length_from_histogram": 6.275862068965517, "spec_accept_rate": 0.35172413793103446, "spec_accepted_drafts": 153, "spec_proposed_drafts": 435, "spec_accept_histogram": [1, 1, 1, 8, 4, 2, 3, 1, 3, 2, 1, 1, 1], "spec_verify_ct": 29, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "9b9cd642d1ea4a24987b9eef643f92c6", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 732, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171038.1344798, "queue_time": 0.00012922286987304688, "reasoning_tokens": 0, "completion_tokens": 181, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 181}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 9.535634547006339, "spec_accept_rate": 0.35172413793103446, "spec_accept_length": 6.241379310344827, "spec_num_correct_drafts": 153, "spec_num_proposed_drafts": 435, "spec_verify_ct": 29, "spec_accepted_drafts": 153, "spec_proposed_drafts": 435, "spec_correct_drafts_histogram": [1, 1, 1, 8, 4, 2, 3, 1, 3, 2, 1, 1, 1], "spec_accept_histogram": [1, 1, 1, 8, 4, 2, 3, 1, 3, 2, 1, 1, 1], "request_received_ts": 1788171034.9309604, "api_server_dispatch_finish_ts": 1788171034.9341989, "request_finished_ts": 1788171044.4665952, "decode_throughput": 56298726.5715118, "response_sent_to_client_ts": 1788171044.4673266}} +{"prompt_id": 20, "output": " There are 15 x 3/5 = <<15*3/5=9>>9 liters of water from the 15 liters pineapple drink.\nAfter 1 liter of orange drink was spilled, there were 10 - 1 = <<10-1=9>>9 liters of orange drink left.\nOut of the 9 liters, 9 x 2/3 = <<9*2/3=6>>6 liters are water.\nThus, there are a total of 9 + 6 = <<9+6=15>>15 liters of water out of the 24 liters.\n#### 15\n\n", "correct": true, "error": null, "completion_tokens": 128, "spec_accept_length": 5.818181818181818, "spec_accept_length_from_histogram": 5.863636363636363, "spec_accept_rate": 0.3242424242424242, "spec_accepted_drafts": 107, "spec_proposed_drafts": 330, "spec_accept_histogram": [1, 1, 3, 5, 0, 7, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 22, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "14c4803b2d744c588ea0d9c6b6a28f40", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171038.3365996, "queue_time": 0.0001316070556640625, "reasoning_tokens": 0, "completion_tokens": 128, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 128}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 8.020555406808853, "spec_accept_rate": 0.3242424242424242, "spec_accept_length": 5.818181818181818, "spec_num_correct_drafts": 107, "spec_num_proposed_drafts": 330, "spec_verify_ct": 22, "spec_accepted_drafts": 107, "spec_proposed_drafts": 330, "spec_correct_drafts_histogram": [1, 1, 3, 5, 0, 7, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [1, 1, 3, 5, 0, 7, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788171035.6247268, "api_server_dispatch_finish_ts": 1788171035.6279175, "request_finished_ts": 1788171043.645282, "decode_throughput": 40.777434737710614, "response_sent_to_client_ts": 1788171043.6458201}} +{"prompt_id": 21, "output": " When Raymond's son was born Samantha was 23 - 6 = <<23-6=17>>17 years old.\nThus it has been 31 - 17 = <<31-17=14>>14 years since Raymond's son was born.\n#### 14\n\n", "correct": true, "error": null, "completion_tokens": 56, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 7.125, "spec_accept_rate": 0.4083333333333333, "spec_accepted_drafts": 49, "spec_proposed_drafts": 120, "spec_accept_histogram": [0, 0, 0, 2, 0, 3, 0, 0, 2, 0, 0, 0, 1], "spec_verify_ct": 8, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "ea994b86f889459aba915a3ba2c4d988", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 711, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171040.3478446, "queue_time": 0.00013136863708496094, "reasoning_tokens": 0, "completion_tokens": 56, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 56}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.044767498970032, "spec_accept_rate": 0.4083333333333333, "spec_accept_length": 7.0, "spec_num_correct_drafts": 49, "spec_num_proposed_drafts": 120, "spec_verify_ct": 8, "spec_accepted_drafts": 49, "spec_proposed_drafts": 120, "spec_correct_drafts_histogram": [0, 0, 0, 2, 0, 3, 0, 0, 2, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 0, 2, 0, 3, 0, 0, 2, 0, 0, 0, 1], "request_received_ts": 1788171037.7175963, "api_server_dispatch_finish_ts": 1788171037.7211146, "request_finished_ts": 1788171042.7623637, "decode_throughput": 14612346.980081653, "response_sent_to_client_ts": 1788171042.7628555}} +{"prompt_id": 22, "output": " His first 3 customers buy 3 * 1 = <<3*1=3>>3 DVDs.\nHis next 2 buy 2 * 2 = <<2*2=4>>4 DVDs.\nHe sells a total of 3 + 4 + 0 = <<3+4+0=7>>7 DVDs.\n#### 7\n\n", "correct": true, "error": null, "completion_tokens": 74, "spec_accept_length": 8.222222222222221, "spec_accept_length_from_histogram": 8.333333333333332, "spec_accept_rate": 0.4888888888888889, "spec_accepted_drafts": 66, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 0, 1, 1, 1, 0, 0, 0, 1, 3, 1, 0, 1], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "34c9312202904c7bad1e8a638b797b9e", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 722, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171040.5366993, "queue_time": 0.00018906593322753906, "reasoning_tokens": 0, "completion_tokens": 74, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 74}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.506346830166876, "spec_accept_rate": 0.4888888888888889, "spec_accept_length": 8.222222222222221, "spec_num_correct_drafts": 66, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 66, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 0, 1, 1, 1, 0, 0, 0, 1, 3, 1, 0, 1], "spec_accept_histogram": [0, 0, 1, 1, 1, 0, 0, 0, 1, 3, 1, 0, 1], "request_received_ts": 1788171037.7225342, "api_server_dispatch_finish_ts": 1788171037.7254233, "request_finished_ts": 1788171043.2288811, "decode_throughput": 21634875.28346674, "response_sent_to_client_ts": 1788171043.2296798}} +{"prompt_id": 23, "output": " The candle burns for 5 - 1 = <<5-1=4>>4 hours.\nThus, the candle will be 2 * 4 = <<2*4=8>>8 centimeters shorter.\n#### 8\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 6.125, "spec_accept_length_from_histogram": 6.25, "spec_accept_rate": 0.35, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_accept_histogram": [0, 0, 1, 1, 2, 1, 0, 1, 1, 1], "spec_verify_ct": 8, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "29a98b669d774f0c85e8d62849cddf5a", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 706, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171040.7640514, "queue_time": 0.00013399124145507812, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.265191392973065, "spec_accept_rate": 0.35, "spec_accept_length": 6.125, "spec_num_correct_drafts": 42, "spec_num_proposed_drafts": 120, "spec_verify_ct": 8, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_correct_drafts_histogram": [0, 0, 1, 1, 2, 1, 0, 1, 1, 1], "spec_accept_histogram": [0, 0, 1, 1, 2, 1, 0, 1, 1, 1], "request_received_ts": 1788171037.964066, "api_server_dispatch_finish_ts": 1788171037.967336, "request_finished_ts": 1788171043.2292573, "decode_throughput": 24910395.143547606, "response_sent_to_client_ts": 1788171043.2308676}} +{"prompt_id": 24, "output": " Let X be the original price of the book. The discounted price is X - X*25% = $19.50.\nCombining like terms, we get .75X = $19.50\nDividing both sides by .75, we get X = $26.\n#### 26\n\n", "correct": true, "error": null, "completion_tokens": 63, "spec_accept_length": 4.846153846153846, "spec_accept_length_from_histogram": 4.923076923076923, "spec_accept_rate": 0.26153846153846155, "spec_accepted_drafts": 51, "spec_proposed_drafts": 195, "spec_accept_histogram": [1, 0, 3, 4, 1, 1, 1, 0, 1, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "a26b8f007ffd4104be14fea17857003c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 703, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171041.6730297, "queue_time": 0.0002231597900390625, "reasoning_tokens": 0, "completion_tokens": 63, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 63}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.050237540155649, "spec_accept_rate": 0.26153846153846155, "spec_accept_length": 4.846153846153846, "spec_num_correct_drafts": 51, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 51, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [1, 0, 3, 4, 1, 1, 1, 0, 1, 0, 1], "spec_accept_histogram": [1, 0, 3, 4, 1, 1, 1, 0, 1, 0, 1], "request_received_ts": 1788171039.4167023, "api_server_dispatch_finish_ts": 1788171039.4199271, "request_finished_ts": 1788171044.4669397, "decode_throughput": 30963717.715348836, "response_sent_to_client_ts": 1788171044.4682224}} +{"prompt_id": 25, "output": " Five packs of milk cost $3 x 5 = $<<3*5=15>>15.\nFour apples cost $1.50 x 4 = $<<1.5*4=6>>6.\nThe total cost of the chicken meal, milk, and apples is $12 + $15 + $6 = $<<12+15+6=33>>33.\nThus, the boxes of pizza cost $50 - $33 = $<<50-33=17>>17.\nTherefore, Marie ordered $17/$8.50 = <<17/8.5=2>>2 boxes of pizza.\n#### 2\n\n", "correct": true, "error": null, "completion_tokens": 131, "spec_accept_length": 6.894736842105263, "spec_accept_length_from_histogram": 6.947368421052632, "spec_accept_rate": 0.39649122807017545, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_accept_histogram": [0, 1, 2, 3, 2, 2, 1, 3, 0, 1, 3, 0, 0, 0, 0, 1], "spec_verify_ct": 19, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5b4bff694e1d4f628743b9930f7bb727", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 731, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171043.4438434, "queue_time": 0.00015735626220703125, "reasoning_tokens": 0, "completion_tokens": 131, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 131}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.790783246979117, "spec_accept_rate": 0.39649122807017545, "spec_accept_length": 6.894736842105263, "spec_num_correct_drafts": 113, "spec_num_proposed_drafts": 285, "spec_verify_ct": 19, "spec_accepted_drafts": 113, "spec_proposed_drafts": 285, "spec_correct_drafts_histogram": [0, 1, 2, 3, 2, 2, 1, 3, 0, 1, 3, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 3, 2, 2, 1, 3, 0, 1, 3, 0, 0, 0, 0, 1], "request_received_ts": 1788171041.3213995, "api_server_dispatch_finish_ts": 1788171041.3247201, "request_finished_ts": 1788171046.1121826, "decode_throughput": 37099385.281063125, "response_sent_to_client_ts": 1788171046.1126842}} +{"prompt_id": 26, "output": " 3 * (16.50 + 22.50 + 42) = $<<3*(16.50+22.50+42)=243>>243\nMishka spent $243 on clothing.\n#### 243\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 7.142857142857143, "spec_accept_rate": 0.4095238095238095, "spec_accepted_drafts": 43, "spec_proposed_drafts": 105, "spec_accept_histogram": [0, 1, 0, 0, 1, 0, 1, 2, 0, 2], "spec_verify_ct": 7, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "db02870e04c24dc1989d6dfc3fa91b80", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 731, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171044.3085938, "queue_time": 0.0001575946807861328, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.7027414380572736, "spec_accept_rate": 0.4095238095238095, "spec_accept_length": 7.0, "spec_num_correct_drafts": 43, "spec_num_proposed_drafts": 105, "spec_verify_ct": 7, "spec_accepted_drafts": 43, "spec_proposed_drafts": 105, "spec_correct_drafts_histogram": [0, 1, 0, 0, 1, 0, 1, 2, 0, 2], "spec_accept_histogram": [0, 1, 0, 0, 1, 0, 1, 2, 0, 2], "request_received_ts": 1788171041.698959, "api_server_dispatch_finish_ts": 1788171041.7022228, "request_finished_ts": 1788171045.4017005, "decode_throughput": 15941728.2870399, "response_sent_to_client_ts": 1788171045.4021764}} +{"prompt_id": 27, "output": " Each container of ice cream has 15 servings and she eats 1 serving a night so after 60 days she will need 60/15 = 4 containers of ice cream\nIf each carton costs $4.00 and she needs 4 containers then it will cost her 4*4 = $<<4*4=16.00>>16.00\n#### 16\n\n", "correct": true, "error": null, "completion_tokens": 80, "spec_accept_length": 5.333333333333333, "spec_accept_length_from_histogram": 5.4, "spec_accept_rate": 0.29333333333333333, "spec_accepted_drafts": 66, "spec_proposed_drafts": 225, "spec_accept_histogram": [0, 1, 2, 3, 4, 2, 0, 2, 0, 0, 0, 0, 1], "spec_verify_ct": 15, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "b40f89caf08944f0b320386fc37b702d", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 721, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171045.6396155, "queue_time": 0.00026488304138183594, "reasoning_tokens": 0, "completion_tokens": 80, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 80}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.235636665020138, "spec_accept_rate": 0.29333333333333333, "spec_accept_length": 5.333333333333333, "spec_num_correct_drafts": 66, "spec_num_proposed_drafts": 225, "spec_verify_ct": 15, "spec_accepted_drafts": 66, "spec_proposed_drafts": 225, "spec_correct_drafts_histogram": [0, 1, 2, 3, 4, 2, 0, 2, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 3, 4, 2, 0, 2, 0, 0, 0, 0, 1], "request_received_ts": 1788171042.7703478, "api_server_dispatch_finish_ts": 1788171042.7749627, "request_finished_ts": 1788171048.0059845, "decode_throughput": 26852043.08198797, "response_sent_to_client_ts": 1788171048.0064573}} +{"prompt_id": 28, "output": " He traveled 20 miles + 15 miles = <<20+15=35>>35 miles not counting the distance between stops.\nHenry traveled 60 miles - 35 miles = <<60-35=25>>25 miles between his first and second stop.\n#### 25\n\n", "correct": true, "error": null, "completion_tokens": 57, "spec_accept_length": 5.7, "spec_accept_length_from_histogram": 5.8, "spec_accept_rate": 0.32, "spec_accepted_drafts": 48, "spec_proposed_drafts": 150, "spec_accept_histogram": [0, 2, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 10, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "8caaaceb5c01459cae365a46aae9db79", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 715, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171046.2403767, "queue_time": 0.0001342296600341797, "reasoning_tokens": 0, "completion_tokens": 57, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 57}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.5570953781716526, "spec_accept_rate": 0.32, "spec_accept_length": 5.7, "spec_num_correct_drafts": 48, "spec_num_proposed_drafts": 150, "spec_verify_ct": 10, "spec_accepted_drafts": 48, "spec_proposed_drafts": 150, "spec_correct_drafts_histogram": [0, 2, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 2, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788171043.2511187, "api_server_dispatch_finish_ts": 1788171043.2540686, "request_finished_ts": 1788171047.808214, "decode_throughput": 16821805.047978736, "response_sent_to_client_ts": 1788171047.808872}} +{"prompt_id": 29, "output": " The second pair of heels costs 33 * 2 = $<<33*2=66>>66.\nThe heels together cost 66 + 33 = $<<66+33=99>>99.\nThe boots cost $5 more than both pairs of heels together, so the boots cost 99 + 5 = $104.\n#### 104\n\n", "correct": true, "error": null, "completion_tokens": 73, "spec_accept_length": 6.636363636363637, "spec_accept_length_from_histogram": 6.7272727272727275, "spec_accept_rate": 0.38181818181818183, "spec_accepted_drafts": 63, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 1, 2, 1, 2, 1, 1, 0, 2, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "da7cb0b778d542c48f4a2d96d6de41fa", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 734, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171046.5084116, "queue_time": 0.00026297569274902344, "reasoning_tokens": 0, "completion_tokens": 73, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 73}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.372211418114603, "spec_accept_rate": 0.38181818181818183, "spec_accept_length": 6.636363636363637, "spec_num_correct_drafts": 63, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 63, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 1, 2, 1, 2, 1, 1, 0, 2, 1], "spec_accept_histogram": [0, 0, 1, 2, 1, 2, 1, 1, 0, 2, 1], "request_received_ts": 1788171043.2466784, "api_server_dispatch_finish_ts": 1788171043.2501311, "request_finished_ts": 1788171048.6188898, "decode_throughput": 88.8612370281484, "response_sent_to_client_ts": 1788171048.6194963}} +{"prompt_id": 30, "output": " The total ratio representing their ages is 7+11= <<7+11=18>>18\nSince the fraction of the ratio that represents Allen's age is 11/18, Allen's current age is 11/18*162 = <<11/18*162=99>>99\nIf Allen is currently 99 years old, in 10 years he will be 99+10 = <<99+10=109>>109 years old\n#### 109\n\n", "correct": true, "error": null, "completion_tokens": 98, "spec_accept_length": 6.125, "spec_accept_length_from_histogram": 6.1875, "spec_accept_rate": 0.3458333333333333, "spec_accepted_drafts": 83, "spec_proposed_drafts": 240, "spec_accept_histogram": [0, 0, 3, 3, 2, 3, 0, 0, 3, 0, 1, 1], "spec_verify_ct": 16, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "72d1a93ea65f451685a63ea38b9cc34c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 703, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171046.9418125, "queue_time": 0.0001277923583984375, "reasoning_tokens": 0, "completion_tokens": 98, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 98}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.547496572136879, "spec_accept_rate": 0.3458333333333333, "spec_accept_length": 6.125, "spec_num_correct_drafts": 83, "spec_num_proposed_drafts": 240, "spec_verify_ct": 16, "spec_accepted_drafts": 83, "spec_proposed_drafts": 240, "spec_correct_drafts_histogram": [0, 0, 3, 3, 2, 3, 0, 0, 3, 0, 1, 1], "spec_accept_histogram": [0, 0, 3, 3, 2, 3, 0, 0, 3, 0, 1, 1], "request_received_ts": 1788171043.655841, "api_server_dispatch_finish_ts": 1788171043.6590207, "request_finished_ts": 1788171050.2033377, "decode_throughput": 61.23421026675583, "response_sent_to_client_ts": 1788171050.2038069}} +{"prompt_id": 31, "output": " The second person guessed 60 because 20 + (80/2)= <<20+(80/2)=60>>60\nThe second guessed 100 because 80 x 1.25 = <<80*1.25=100>>100\nThe total amount guessed is 240 because 80 + 60 + 100= <<80+60+100=240>>240\nThe average amount guessed is 80 because 240 / 3 = <<240/3=80>>80\n#### 80\n\n", "correct": true, "error": null, "completion_tokens": 107, "spec_accept_length": 5.944444444444445, "spec_accept_length_from_histogram": 5.888888888888889, "spec_accept_rate": 0.32592592592592595, "spec_accepted_drafts": 88, "spec_proposed_drafts": 270, "spec_accept_histogram": [1, 2, 5, 2, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1], "spec_verify_ct": 18, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "20dd7ef70fea4961971b170c798a9200", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171047.9902349, "queue_time": 0.000263214111328125, "reasoning_tokens": 0, "completion_tokens": 107, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 107}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.1681548319756985, "spec_accept_rate": 0.32592592592592595, "spec_accept_length": 5.944444444444445, "spec_num_correct_drafts": 88, "spec_num_proposed_drafts": 270, "spec_verify_ct": 18, "spec_accepted_drafts": 88, "spec_proposed_drafts": 270, "spec_correct_drafts_histogram": [1, 2, 5, 2, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1], "spec_accept_histogram": [1, 2, 5, 2, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1], "request_received_ts": 1788171044.4829614, "api_server_dispatch_finish_ts": 1788171044.4861844, "request_finished_ts": 1788171051.6511161, "decode_throughput": 28803399.555611793, "response_sent_to_client_ts": 1788171051.652002}} +{"prompt_id": 32, "output": " He spends 10*.5=<<10*.5=5>>5 hours per day\nThat means he spends 5*7=<<5*7=35>>35 hours per week\n#### 35\n\n", "correct": true, "error": null, "completion_tokens": 45, "spec_accept_length": 6.428571428571429, "spec_accept_length_from_histogram": 6.571428571428571, "spec_accept_rate": 0.37142857142857144, "spec_accepted_drafts": 39, "spec_proposed_drafts": 105, "spec_accept_histogram": [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "spec_verify_ct": 7, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "7efc78a27fe8484ebe2802a59cc8711e", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 708, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171047.9902349, "queue_time": 0.0001518726348876953, "reasoning_tokens": 0, "completion_tokens": 45, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 45}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.941901796031743, "spec_accept_rate": 0.37142857142857144, "spec_accept_length": 6.428571428571429, "spec_num_correct_drafts": 39, "spec_num_proposed_drafts": 105, "spec_verify_ct": 7, "spec_accepted_drafts": 39, "spec_proposed_drafts": 105, "spec_correct_drafts_histogram": [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "spec_accept_histogram": [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "request_received_ts": 1788171044.487551, "api_server_dispatch_finish_ts": 1788171044.4903896, "request_finished_ts": 1788171049.429453, "decode_throughput": 14492220.937423313, "response_sent_to_client_ts": 1788171049.4299169}} +{"prompt_id": 33, "output": " Let x be the number of silver coins Gretchen has\nGretchen has x+30 gold coins.\nx+x+30=110\n2*x=80\nx=<<40=40>>40\nGretchen has 40+30=<<40+30=70>>70 gold coins\n#### 70\n\n", "correct": true, "error": null, "completion_tokens": 68, "spec_accept_length": 5.230769230769231, "spec_accept_length_from_histogram": 5.230769230769231, "spec_accept_rate": 0.28205128205128205, "spec_accepted_drafts": 55, "spec_proposed_drafts": 195, "spec_accept_histogram": [1, 3, 2, 2, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c1f9e546172442c2bed3cf0b042ab5f0", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 695, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171048.8013089, "queue_time": 0.0001614093780517578, "reasoning_tokens": 0, "completion_tokens": 68, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 68}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.015332517214119, "spec_accept_rate": 0.28205128205128205, "spec_accept_length": 5.230769230769231, "spec_num_correct_drafts": 55, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 55, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [1, 3, 2, 2, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "spec_accept_histogram": [1, 3, 2, 2, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], "request_received_ts": 1788171045.4122992, "api_server_dispatch_finish_ts": 1788171045.415459, "request_finished_ts": 1788171051.4276316, "decode_throughput": 13131459.744090537, "response_sent_to_client_ts": 1788171051.428137}} +{"prompt_id": 34, "output": " Half of Raymond's jewels is 40/2 = <<40/2=20>>20\nSince Aaron has 5 more jewels than half of Raymond's jewels, he has 20+5 = <<5+20=25>>25 jewels.\nIf Siobhan has 2 fewer jewels than Aaron, he has 25-2 = 23 jewels.\n#### 23\n\n", "correct": true, "error": null, "completion_tokens": 79, "spec_accept_length": 6.583333333333333, "spec_accept_length_from_histogram": 6.666666666666667, "spec_accept_rate": 0.37777777777777777, "spec_accepted_drafts": 68, "spec_proposed_drafts": 180, "spec_accept_histogram": [0, 1, 3, 0, 2, 2, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 12, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3ddecfb4a48b4f06950180eb31fe49be", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 708, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171049.0135803, "queue_time": 0.00013375282287597656, "reasoning_tokens": 0, "completion_tokens": 79, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 79}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.525186281185597, "spec_accept_rate": 0.37777777777777777, "spec_accept_length": 6.583333333333333, "spec_num_correct_drafts": 68, "spec_num_proposed_drafts": 180, "spec_verify_ct": 12, "spec_accepted_drafts": 68, "spec_proposed_drafts": 180, "spec_correct_drafts_histogram": [0, 1, 3, 0, 2, 2, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 3, 0, 2, 2, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788171046.1263468, "api_server_dispatch_finish_ts": 1788171046.129568, "request_finished_ts": 1788171051.6515331, "decode_throughput": 29324881.74789916, "response_sent_to_client_ts": 1788171051.6529446}} +{"prompt_id": 35, "output": " He scored 4*.25=<<4*.25=1>>1 point more in the second 20 minutes\nSo he scored 4+1=<<4+1=5>>5 points in the second 20 minutes\nSo in total, he scored 4+5=<<4+5=9>>9 points in the 40 minutes\n#### 9\n\n", "correct": true, "error": null, "completion_tokens": 79, "spec_accept_length": 8.777777777777779, "spec_accept_length_from_histogram": 8.88888888888889, "spec_accept_rate": 0.5259259259259259, "spec_accepted_drafts": 71, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 0, 1, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 1], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "f53ccf046023492d884a37ed0110adba", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 716, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171050.3849766, "queue_time": 0.0001289844512939453, "reasoning_tokens": 0, "completion_tokens": 79, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 79}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.338141418993473, "spec_accept_rate": 0.5259259259259259, "spec_accept_length": 8.777777777777779, "spec_num_correct_drafts": 71, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 71, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 0, 1, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 1], "request_received_ts": 1788171047.8256965, "api_server_dispatch_finish_ts": 1788171047.828871, "request_finished_ts": 1788171052.163838, "decode_throughput": 152.32085040947746, "response_sent_to_client_ts": 1788171052.16436}} +{"prompt_id": 36, "output": " He eats 2 yogurts a day so over 30 days he will need 2*30 = <<2*30=60>>60 yogurts\nThe yogurt is sold as 4 units for $5.00 and he needs 60 yogurts so that’s 60/4 = <<60/4=15>>15 4-packs\nEach 4-pack is $5.00 and he needs 15 4-packs so he will spend $5*15 = $<<5*15=75.00>>75.00 on yogurt\n#### 75\n\n", "correct": true, "error": null, "completion_tokens": 124, "spec_accept_length": 5.636363636363637, "spec_accept_length_from_histogram": 5.681818181818182, "spec_accept_rate": 0.31212121212121213, "spec_accepted_drafts": 103, "spec_proposed_drafts": 330, "spec_accept_histogram": [0, 2, 4, 3, 3, 2, 2, 1, 4, 0, 0, 1], "spec_verify_ct": 22, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "75b97163a774476081e53fabfe563eb3", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 709, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171050.5958166, "queue_time": 0.00014400482177734375, "reasoning_tokens": 0, "completion_tokens": 124, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 124}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.386951171793044, "spec_accept_rate": 0.31212121212121213, "spec_accept_length": 5.636363636363637, "spec_num_correct_drafts": 103, "spec_num_proposed_drafts": 330, "spec_verify_ct": 22, "spec_accepted_drafts": 103, "spec_proposed_drafts": 330, "spec_correct_drafts_histogram": [0, 2, 4, 3, 3, 2, 2, 1, 4, 0, 0, 1], "spec_accept_histogram": [0, 2, 4, 3, 3, 2, 2, 1, 4, 0, 0, 1], "request_received_ts": 1788171048.017575, "api_server_dispatch_finish_ts": 1788171048.0207539, "request_finished_ts": 1788171054.4045262, "decode_throughput": 63.18343324451378, "response_sent_to_client_ts": 1788171054.4049995}} +{"prompt_id": 37, "output": " He spent $160 on video games because 8 x 20 = <<8*20=160>>160\nHe earned $165 from his sale because 160 + 5 = <<160+5=165>>165\nHe sold 11 lego sets because 11 x 15 = <<11*15=165>>165\nHe has 2 sets left because 13 - 11 = <<13-11=2>>2\n#### 2\n\n", "correct": true, "error": null, "completion_tokens": 96, "spec_accept_length": 8.0, "spec_accept_length_from_histogram": 8.083333333333332, "spec_accept_rate": 0.4722222222222222, "spec_accepted_drafts": 85, "spec_proposed_drafts": 180, "spec_accept_histogram": [0, 1, 1, 1, 1, 0, 1, 2, 0, 2, 0, 2, 0, 0, 0, 1], "spec_verify_ct": 12, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "677e55f1cd34432c9d391ddbe87fe2cb", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171051.8385348, "queue_time": 0.0001857280731201172, "reasoning_tokens": 0, "completion_tokens": 96, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 96}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.970573452766985, "spec_accept_rate": 0.4722222222222222, "spec_accept_length": 8.0, "spec_num_correct_drafts": 85, "spec_num_proposed_drafts": 180, "spec_verify_ct": 12, "spec_accepted_drafts": 85, "spec_proposed_drafts": 180, "spec_correct_drafts_histogram": [0, 1, 1, 1, 1, 0, 1, 2, 0, 2, 0, 2, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 1, 1, 1, 0, 1, 2, 0, 2, 0, 2, 0, 0, 0, 1], "request_received_ts": 1788171048.6297784, "api_server_dispatch_finish_ts": 1788171048.6329646, "request_finished_ts": 1788171053.6003518, "decode_throughput": 31056621.488811083, "response_sent_to_client_ts": 1788171053.6008267}} +{"prompt_id": 38, "output": " He has 7-1=<<7-1=6>>6 possible running days.\nHe runs half of those so he runs 6/2=<<6/2=3>>3 days.\nSo for two days, he runs 3/2=<<3/2=1.5>>1.5 hours.\nSo in total, he runs 3+1.5+1.5=<<3+1.5+1.5=6>>6 hours.\nSo he runs 60/6=<<60/6=10>>10 mph.\n#### 10\n\n", "correct": true, "error": null, "completion_tokens": 124, "spec_accept_length": 6.2, "spec_accept_length_from_histogram": 6.25, "spec_accept_rate": 0.35, "spec_accepted_drafts": 105, "spec_proposed_drafts": 300, "spec_accept_histogram": [0, 3, 1, 2, 3, 0, 4, 3, 1, 1, 2], "spec_verify_ct": 20, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "ab7a2157393646e8acc7044b741705d4", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 711, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171052.707009, "queue_time": 0.00020170211791992188, "reasoning_tokens": 0, "completion_tokens": 124, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 124}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.140904388856143, "spec_accept_rate": 0.35, "spec_accept_length": 6.2, "spec_num_correct_drafts": 105, "spec_num_proposed_drafts": 300, "spec_verify_ct": 20, "spec_accepted_drafts": 105, "spec_proposed_drafts": 300, "spec_correct_drafts_histogram": [0, 3, 1, 2, 3, 0, 4, 3, 1, 1, 2], "spec_accept_histogram": [0, 3, 1, 2, 3, 0, 4, 3, 1, 1, 2], "request_received_ts": 1788171049.4399328, "api_server_dispatch_finish_ts": 1788171049.4431252, "request_finished_ts": 1788171056.5808372, "decode_throughput": 31966657.231513977, "response_sent_to_client_ts": 1788171056.5813298}} +{"prompt_id": 39, "output": " If Dana can skip at half the speed she can run, then she can run at 3*2=<<3*2=6>>6 miles per hour.\nAnd since she can run at a speed that is 4 times faster than she can walk, this means she can walk at 6/4=1.5 miles per hour.\nIf two-thirds of the time is spent walking, then she walks for 6*(2/3)=<<6*(2/3)=4>>4 hours.\nIf one-third of the time is spent running, then she runs for 6-4=<<6-4=2>>2 hours.\nThus, she runs for 2 hours at 6 miles per hour, or 2*6=<<2*6=12>>12 miles.\nShe walks for 4 hours at 1.5 miles per hour, or 4*1.5=<<4*1.5=6>>6 miles.\nThus, altogether, she travels 12+6=<<12+6=18>>18 miles.\n#### 18\n\n", "correct": true, "error": null, "completion_tokens": 225, "spec_accept_length": 6.818181818181818, "spec_accept_length_from_histogram": 6.848484848484849, "spec_accept_rate": 0.3898989898989899, "spec_accepted_drafts": 193, "spec_proposed_drafts": 495, "spec_accept_histogram": [0, 2, 3, 6, 7, 2, 2, 1, 1, 3, 2, 0, 1, 0, 2, 1], "spec_verify_ct": 33, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "bee58840fd9e4984affa6bfa18af4b9b", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 740, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171053.443203, "queue_time": 0.0001366138458251953, "reasoning_tokens": 0, "completion_tokens": 225, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 225}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 10.610384341794997, "spec_accept_rate": 0.3898989898989899, "spec_accept_length": 6.818181818181818, "spec_num_correct_drafts": 193, "spec_num_proposed_drafts": 495, "spec_verify_ct": 33, "spec_accepted_drafts": 193, "spec_proposed_drafts": 495, "spec_correct_drafts_histogram": [0, 2, 3, 6, 7, 2, 2, 1, 1, 3, 2, 0, 1, 0, 2, 1], "spec_accept_histogram": [0, 2, 3, 6, 7, 2, 2, 1, 1, 3, 2, 0, 1, 0, 2, 1], "request_received_ts": 1788171050.2139897, "api_server_dispatch_finish_ts": 1788171050.2172062, "request_finished_ts": 1788171060.824374, "decode_throughput": 290.5402074961987, "response_sent_to_client_ts": 1788171060.8248646}} +{"prompt_id": 40, "output": " Ben’s iPhone is 1*2 = <<1*2=2>>2 years old.\nBrandon’s iPhone is 4*2 = <<4*2=8>>8 years old.\n#### 8\n\n", "correct": true, "error": null, "completion_tokens": 48, "spec_accept_length": 6.857142857142857, "spec_accept_length_from_histogram": 6.714285714285714, "spec_accept_rate": 0.38095238095238093, "spec_accepted_drafts": 40, "spec_proposed_drafts": 105, "spec_accept_histogram": [0, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "spec_verify_ct": 7, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "83533f1dd14b41c88be6a275d02080c1", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 718, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171054.5612996, "queue_time": 0.00014495849609375, "reasoning_tokens": 0, "completion_tokens": 48, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 48}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.596388224978, "spec_accept_rate": 0.38095238095238093, "spec_accept_length": 6.857142857142857, "spec_num_correct_drafts": 40, "spec_num_proposed_drafts": 105, "spec_verify_ct": 7, "spec_accepted_drafts": 40, "spec_proposed_drafts": 105, "spec_correct_drafts_histogram": [0, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "request_received_ts": 1788171051.438173, "api_server_dispatch_finish_ts": 1788171051.4416385, "request_finished_ts": 1788171056.0345612, "decode_throughput": 13143863.973954942, "response_sent_to_client_ts": 1788171056.0350468}} +{"prompt_id": 41, "output": " With the gemstone, Polly could throw the javelin 3 times farther than 400 feet, for a distance of 3*400=<<3*400=1200>>1200 feet.\n1200 feet is beyond the 1000-foot reach of the dragon's flames by a distance of 1200-1000=<<1200-1000=200>>200 feet.\n#### 200\n\n", "correct": true, "error": null, "completion_tokens": 86, "spec_accept_length": 6.615384615384615, "spec_accept_length_from_histogram": 6.6923076923076925, "spec_accept_rate": 0.37948717948717947, "spec_accepted_drafts": 74, "spec_proposed_drafts": 195, "spec_accept_histogram": [0, 1, 1, 2, 0, 2, 3, 0, 1, 2, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "47c78529369843bbabd095943e632cb0", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 794, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171054.7348893, "queue_time": 0.00013637542724609375, "reasoning_tokens": 0, "completion_tokens": 86, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 86}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.367660026997328, "spec_accept_rate": 0.37948717948717947, "spec_accept_length": 6.615384615384615, "spec_num_correct_drafts": 74, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 74, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [0, 1, 1, 2, 0, 2, 3, 0, 1, 2, 0, 1], "spec_accept_histogram": [0, 1, 1, 2, 0, 2, 3, 0, 1, 2, 0, 1], "request_received_ts": 1788171051.6692016, "api_server_dispatch_finish_ts": 1788171051.6726334, "request_finished_ts": 1788171058.0368617, "decode_throughput": 28909741.85619259, "response_sent_to_client_ts": 1788171058.0373404}} +{"prompt_id": 42, "output": " To start the evening, there were 5 pies, each with 8 pieces, which is 5*8=<<5*8=40>>40 pieces of pie.\nIf only 14 remained, then 40-14=<<40-14=26>>26 pieces of pie had been taken by guests.\n#### 26\n\n", "correct": true, "error": null, "completion_tokens": 70, "spec_accept_length": 7.0, "spec_accept_length_from_histogram": 7.1, "spec_accept_rate": 0.4066666666666667, "spec_accepted_drafts": 61, "spec_proposed_drafts": 150, "spec_accept_histogram": [0, 0, 1, 2, 1, 0, 1, 2, 0, 2, 0, 1], "spec_verify_ct": 10, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "a53b456bfc23438e9a2e67b9df953660", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 746, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171054.899614, "queue_time": 0.00020623207092285156, "reasoning_tokens": 0, "completion_tokens": 70, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 70}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.911694198381156, "spec_accept_rate": 0.4066666666666667, "spec_accept_length": 7.0, "spec_num_correct_drafts": 61, "spec_num_proposed_drafts": 150, "spec_verify_ct": 10, "spec_accepted_drafts": 61, "spec_proposed_drafts": 150, "spec_correct_drafts_histogram": [0, 0, 1, 2, 1, 0, 1, 2, 0, 2, 0, 1], "spec_accept_histogram": [0, 0, 1, 2, 1, 0, 1, 2, 0, 2, 0, 1], "request_received_ts": 1788171051.6740005, "api_server_dispatch_finish_ts": 1788171051.6771235, "request_finished_ts": 1788171057.5856948, "decode_throughput": 21574893.959231216, "response_sent_to_client_ts": 1788171057.5862522}} +{"prompt_id": 43, "output": " If the total calorie target is 2000 and I have consumed 1800 calories then I have 2000-1800 = <<2000-1800=200>>200 calories left to eat\nIf each serving of chips has 250 calories and I only have 200 calories left to eat, then I can only eat 200/250 of a serving = 4/5 of a serving\nWe also know that a 300g bag of chips has 5 servings, hence each serving has 300g/5 = <<300/5=60>>60 grams\nIf I can only eat 4/5 of a serving, then I can eat only 60g * 4/5 = 240g/5 = 48 grams\n#### 48\n\n", "correct": true, "error": null, "completion_tokens": 161, "spec_accept_length": 6.1923076923076925, "spec_accept_length_from_histogram": 6.230769230769231, "spec_accept_rate": 0.3487179487179487, "spec_accepted_drafts": 136, "spec_proposed_drafts": 390, "spec_accept_histogram": [0, 0, 6, 3, 3, 1, 5, 3, 2, 1, 0, 2], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c324b46d95b64ce485016ba65bfcb773", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 721, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171055.0815876, "queue_time": 0.00018906593322753906, "reasoning_tokens": 0, "completion_tokens": 161, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 161}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 9.562264379113913, "spec_accept_rate": 0.3487179487179487, "spec_accept_length": 6.1923076923076925, "spec_num_correct_drafts": 136, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 136, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [0, 0, 6, 3, 3, 1, 5, 3, 2, 1, 0, 2], "spec_accept_histogram": [0, 0, 6, 3, 3, 1, 5, 3, 2, 1, 0, 2], "request_received_ts": 1788171052.1747992, "api_server_dispatch_finish_ts": 1788171052.1780963, "request_finished_ts": 1788171061.7370636, "decode_throughput": 52.739358904895894, "response_sent_to_client_ts": 1788171061.7374933}} +{"prompt_id": 44, "output": " 1 pound of beeswax and the wick make 10 candles and he wants to make 20 candles so he needs 20/10 = <<20/10=2>>2 pounds of beeswax\n1 pound of beeswax costs $10.00 and he needs 2 pounds so his supplies will cost 10*2 = $<<10*2=20.00>>20.00\nHe wants to sell each candle for $2.00 each and he's selling 20 candles so that's 2*20 = $<<2*20=40.00>>40.00\nIf he makes $40.00 selling candles and his supplies cost $20.00 then his net profit is 40-20 = $<<40-20=20.00>>20.00\n#### 20\n\n", "correct": true, "error": null, "completion_tokens": 168, "spec_accept_length": 6.461538461538462, "spec_accept_length_from_histogram": 6.576923076923077, "spec_accept_rate": 0.3717948717948718, "spec_accepted_drafts": 145, "spec_proposed_drafts": 390, "spec_accept_histogram": [1, 0, 5, 2, 2, 2, 4, 4, 2, 0, 2, 1, 1], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "16aa005c2589494da80b65ec689d851f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 737, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171056.2749515, "queue_time": 0.00014138221740722656, "reasoning_tokens": 0, "completion_tokens": 168, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 168}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 9.147357358131558, "spec_accept_rate": 0.3717948717948718, "spec_accept_length": 6.461538461538462, "spec_num_correct_drafts": 145, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 145, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [1, 0, 5, 2, 2, 2, 4, 4, 2, 0, 2, 1, 1], "spec_accept_histogram": [1, 0, 5, 2, 2, 2, 4, 4, 2, 0, 2, 1, 1], "request_received_ts": 1788171053.611392, "api_server_dispatch_finish_ts": 1788171053.614705, "request_finished_ts": 1788171062.7587492, "decode_throughput": 50390581.59561613, "response_sent_to_client_ts": 1788171062.7595296}} +{"prompt_id": 45, "output": " If she wrote five articles on Monday, then on Tuesday she wrote 2/5* 5 = 2 more articles.\nThe total number of articles she wrote on Tuesday is 5+2 = <<5+2=7>>7\nOn Wednesday, the number of articles was twice the number she wrote on Tuesday, so it was 2*7 = <<2*7=14>>14 articles.\nOver the three days, she wrote 5+7 +14 = <<5+7+14=26>>26 articles.\nIf each article takes her 4 hours to research and write about, she spent 4*26 = <<4*26=104>>104 hours on all the articles.\n#### 104\n\n", "correct": true, "error": null, "completion_tokens": 150, "spec_accept_length": 6.25, "spec_accept_length_from_histogram": 6.291666666666667, "spec_accept_rate": 0.3527777777777778, "spec_accepted_drafts": 127, "spec_proposed_drafts": 360, "spec_accept_histogram": [0, 0, 2, 5, 5, 3, 1, 1, 4, 2, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "4eec52121b3c432ba26b586363bbbefa", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 760, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171056.8366196, "queue_time": 0.00013518333435058594, "reasoning_tokens": 0, "completion_tokens": 150, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 150}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 8.343348165974021, "spec_accept_rate": 0.3527777777777778, "spec_accept_length": 6.25, "spec_num_correct_drafts": 127, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 127, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [0, 0, 2, 5, 5, 3, 1, 1, 4, 2, 1], "spec_accept_histogram": [0, 0, 2, 5, 5, 3, 1, 1, 4, 2, 1], "request_received_ts": 1788171054.4157987, "api_server_dispatch_finish_ts": 1788171054.4190912, "request_finished_ts": 1788171062.759147, "decode_throughput": 38.772243314189815, "response_sent_to_client_ts": 1788171062.7604156}} +{"prompt_id": 46, "output": " Let's let x be the number of post-it notes in the package that Candice purchased.\nThen, the total number of post-it notes she had when she left the store was 80+x.\nAnd if she used 220 Post-it notes at work and had 23 left, then the total number she had when she left the store was 220+23\nSubtracting 80 from each side of the equation, we see that x=220+23-80=<<220+23-80=163>>163 Post-it notes.\n#### 163\n\n", "correct": true, "error": null, "completion_tokens": 114, "spec_accept_length": 4.956521739130435, "spec_accept_length_from_histogram": 5.0, "spec_accept_rate": 0.26666666666666666, "spec_accepted_drafts": 92, "spec_proposed_drafts": 345, "spec_accept_histogram": [2, 3, 4, 4, 0, 3, 1, 2, 2, 2], "spec_verify_ct": 23, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "210b5cb281f54b84bad2e218aa325340", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 757, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171058.2497754, "queue_time": 0.00016808509826660156, "reasoning_tokens": 0, "completion_tokens": 114, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 114}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.858733362983912, "spec_accept_rate": 0.26666666666666666, "spec_accept_length": 4.956521739130435, "spec_num_correct_drafts": 92, "spec_num_proposed_drafts": 345, "spec_verify_ct": 23, "spec_accepted_drafts": 92, "spec_proposed_drafts": 345, "spec_correct_drafts_histogram": [2, 3, 4, 4, 0, 3, 1, 2, 2, 2], "spec_accept_histogram": [2, 3, 4, 4, 0, 3, 1, 2, 2, 2], "request_received_ts": 1788171056.0451891, "api_server_dispatch_finish_ts": 1788171056.0484638, "request_finished_ts": 1788171063.9039226, "decode_throughput": 37127547.76988984, "response_sent_to_client_ts": 1788171063.9046683}} +{"prompt_id": 47, "output": " He bought 200 / 40 = <<200/40=5>>5 blue ties\nSo he bought 5 * 2 = <<5*2=10>>10 red ties\nEach red tie cost $40 * .5 = $<<40*.5=20>>20 more than blue ties\nSo they each cost $40 + $20 = $<<40+20=60>>60\nThus he spent $60 * 10 = $<<60*10=600>>600 on red ties\nSo he spent $200 + $600 = $<<200+600=800>>800 on ties\n#### 800\n\n", "correct": true, "error": null, "completion_tokens": 131, "spec_accept_length": 6.238095238095238, "spec_accept_length_from_histogram": 6.285714285714286, "spec_accept_rate": 0.3523809523809524, "spec_accepted_drafts": 111, "spec_proposed_drafts": 315, "spec_accept_histogram": [0, 0, 3, 5, 3, 2, 2, 1, 2, 2, 0, 0, 0, 0, 0, 1], "spec_verify_ct": 21, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5ab7906969db41c3846c3b6e2507ceb7", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 714, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171059.1328783, "queue_time": 0.00013518333435058594, "reasoning_tokens": 0, "completion_tokens": 131, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 131}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.916867175139487, "spec_accept_rate": 0.3523809523809524, "spec_accept_length": 6.238095238095238, "spec_num_correct_drafts": 111, "spec_num_proposed_drafts": 315, "spec_verify_ct": 21, "spec_accepted_drafts": 111, "spec_proposed_drafts": 315, "spec_correct_drafts_histogram": [0, 0, 3, 5, 3, 2, 2, 1, 2, 2, 0, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 3, 5, 3, 2, 2, 1, 2, 2, 0, 0, 0, 0, 0, 1], "request_received_ts": 1788171056.5916772, "api_server_dispatch_finish_ts": 1788171056.5948532, "request_finished_ts": 1788171064.5085442, "decode_throughput": 35823543.46721417, "response_sent_to_client_ts": 1788171064.5092762}} +{"prompt_id": 48, "output": " If Tracy used a piece of wire 4 feet long, since 1 foot is equal to 12 inches, the total length of the wire in inches was 12*4=<<12*4=48>>48 inches.\nSince the wire was cut into pieces 6 inches long, Tracy obtained 48/6=<<48/6=8>>8 pieces\n#### 8\n\n", "correct": true, "error": null, "completion_tokens": 81, "spec_accept_length": 6.230769230769231, "spec_accept_length_from_histogram": 6.3076923076923075, "spec_accept_rate": 0.35384615384615387, "spec_accepted_drafts": 69, "spec_proposed_drafts": 195, "spec_accept_histogram": [0, 1, 2, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "0fb45f3af2444261b6a5ab75d287f36c", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 704, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171060.28571, "queue_time": 0.0001742839813232422, "reasoning_tokens": 0, "completion_tokens": 81, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 81}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.307476520072669, "spec_accept_rate": 0.35384615384615387, "spec_accept_length": 6.230769230769231, "spec_num_correct_drafts": 69, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 69, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [0, 1, 2, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 1], "request_received_ts": 1788171057.5967891, "api_server_dispatch_finish_ts": 1788171057.6000016, "request_finished_ts": 1788171063.9042656, "decode_throughput": 37388181.03155604, "response_sent_to_client_ts": 1788171063.9055488}} +{"prompt_id": 49, "output": " The total number of units in the building will be 8 units/floor * 15 floors = <<8*15=120>>120 units.\nIf 3/4 of the building is occupied, then the total number of occupied units is 3/4 * 120 units = <<3/4*120=90>>90 units.\nThe total number of unoccupied units is 120 units - 90 units = <<120-90=30>>30 units.\n#### 30\n\n", "correct": true, "error": null, "completion_tokens": 101, "spec_accept_length": 6.733333333333333, "spec_accept_length_from_histogram": 6.866666666666666, "spec_accept_rate": 0.39111111111111113, "spec_accepted_drafts": 88, "spec_proposed_drafts": 225, "spec_accept_histogram": [0, 2, 1, 2, 1, 1, 3, 2, 0, 1, 0, 0, 0, 1, 0, 1], "spec_verify_ct": 15, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "5c320415d3fa4930b70ba75ab79216c7", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 709, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171060.537147, "queue_time": 0.00012636184692382812, "reasoning_tokens": 0, "completion_tokens": 101, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 101}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.461538522038609, "spec_accept_rate": 0.39111111111111113, "spec_accept_length": 6.733333333333333, "spec_num_correct_drafts": 88, "spec_num_proposed_drafts": 225, "spec_verify_ct": 15, "spec_accepted_drafts": 88, "spec_proposed_drafts": 225, "spec_correct_drafts_histogram": [0, 2, 1, 2, 1, 1, 3, 2, 0, 1, 0, 0, 0, 1, 0, 1], "spec_accept_histogram": [0, 2, 1, 2, 1, 1, 3, 2, 0, 1, 0, 0, 0, 1, 0, 1], "request_received_ts": 1788171058.0473526, "api_server_dispatch_finish_ts": 1788171058.0505698, "request_finished_ts": 1788171064.508891, "decode_throughput": 52569979.14320686, "response_sent_to_client_ts": 1788171064.5101848}} +{"prompt_id": 50, "output": " In a week, Lloyd's egg farm produces 252 x 7 = <<252*7=1764>>1764 eggs.\nThere are 1764/12 = <<1764/12=147>>147 dozens of eggs produced in a week.\nSo, Lloyd makes 147 x $2 = $<<147*2=294>>294 on eggs.\n#### 294\n\n", "correct": true, "error": null, "completion_tokens": 79, "spec_accept_length": 5.642857142857143, "spec_accept_length_from_histogram": 5.714285714285714, "spec_accept_rate": 0.3142857142857143, "spec_accepted_drafts": 66, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 1, 2, 2, 2, 3, 1, 0, 2, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "baefa51a7dae426aaedf1d221729aa86", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 702, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171062.5055192, "queue_time": 0.0001933574676513672, "reasoning_tokens": 0, "completion_tokens": 79, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 79}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.607200863305479, "spec_accept_rate": 0.3142857142857143, "spec_accept_length": 5.642857142857143, "spec_num_correct_drafts": 66, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 66, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 1, 2, 2, 2, 3, 1, 0, 2, 0, 1], "spec_accept_histogram": [0, 1, 2, 2, 2, 3, 1, 0, 2, 0, 1], "request_received_ts": 1788171060.8354845, "api_server_dispatch_finish_ts": 1788171060.838699, "request_finished_ts": 1788171065.4426854, "decode_throughput": 19978974.778625954, "response_sent_to_client_ts": 1788171065.4431963}} +{"prompt_id": 51, "output": " He was travelling at full speed for 4-1=<<4-1=3>>3 hours\nSo he went 3*10=<<3*10=30>>30 miles\nThat means it takes 30/6=<<30/6=5>>5 hours to get back\n#### 5\n\n", "correct": true, "error": null, "completion_tokens": 67, "spec_accept_length": 6.090909090909091, "spec_accept_length_from_histogram": 6.181818181818182, "spec_accept_rate": 0.34545454545454546, "spec_accepted_drafts": 57, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 2, 1, 2, 0, 2, 0, 0, 0, 3, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e052601263e647209696704d5bd87cf8", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 716, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171063.6937482, "queue_time": 0.00014662742614746094, "reasoning_tokens": 0, "completion_tokens": 67, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 67}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.047629018314183, "spec_accept_rate": 0.34545454545454546, "spec_accept_length": 6.090909090909091, "spec_num_correct_drafts": 57, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 57, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 2, 1, 2, 0, 2, 0, 0, 0, 3, 1], "spec_accept_histogram": [0, 2, 1, 2, 0, 2, 0, 0, 0, 3, 1], "request_received_ts": 1788171061.7477229, "api_server_dispatch_finish_ts": 1788171061.7509506, "request_finished_ts": 1788171065.7953517, "decode_throughput": 15158708.103529412, "response_sent_to_client_ts": 1788171065.795834}} +{"prompt_id": 52, "output": " 30 comic books weigh 7.5 pounds because 30 x .25 = <<30*.25=7.5>>7.5\nHe needs to remove 7.5 more pounds because 15 - 7.5 = <<15-7.5=7.5>>7.5\nHe needs to remove 15 toys because 7.5 / .5 = <<15=15>>15\n#### 15\n\n", "correct": true, "error": null, "completion_tokens": 92, "spec_accept_length": 8.363636363636363, "spec_accept_length_from_histogram": 8.454545454545453, "spec_accept_rate": 0.49696969696969695, "spec_accepted_drafts": 82, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 0, 1, 0, 3, 2, 0, 2, 0, 1, 1, 0, 0, 0, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "a5023d117df2456c8376f26f7809898b", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 728, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171065.1119113, "queue_time": 0.00012946128845214844, "reasoning_tokens": 0, "completion_tokens": 92, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 92}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.139327056705952, "spec_accept_rate": 0.49696969696969695, "spec_accept_length": 8.363636363636363, "spec_num_correct_drafts": 82, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 82, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 0, 1, 0, 3, 2, 0, 2, 0, 1, 1, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 0, 1, 0, 3, 2, 0, 2, 0, 1, 1, 0, 0, 0, 1], "request_received_ts": 1788171062.7804968, "api_server_dispatch_finish_ts": 1788171062.7833636, "request_finished_ts": 1788171066.919824, "decode_throughput": 23607273.733752113, "response_sent_to_client_ts": 1788171066.9203212}} +{"prompt_id": 53, "output": " On Thursday the mechanic charged a total of 6*60 + 4*40 = $<<6*60+4*40=520>>520.\nOn Friday the mechanic charged a total of 12*40 = $<<12*40=480>>480.\nThus, on Thursday, the day with higher revenue, the mechanic had 520-480 = $<<520-480=40>>40 more revenue than on Friday.\n#### 40\n\n", "correct": true, "error": null, "completion_tokens": 94, "spec_accept_length": 6.714285714285714, "spec_accept_length_from_histogram": 6.785714285714286, "spec_accept_rate": 0.38571428571428573, "spec_accepted_drafts": 81, "spec_proposed_drafts": 210, "spec_accept_histogram": [0, 0, 1, 5, 0, 1, 2, 1, 1, 1, 1, 0, 0, 1], "spec_verify_ct": 14, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e69798cc347147f6ac632826e58eb37f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 764, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171064.8335547, "queue_time": 0.000213623046875, "reasoning_tokens": 0, "completion_tokens": 94, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 94}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.33417454501614, "spec_accept_rate": 0.38571428571428573, "spec_accept_length": 6.714285714285714, "spec_num_correct_drafts": 81, "spec_num_proposed_drafts": 210, "spec_verify_ct": 14, "spec_accepted_drafts": 81, "spec_proposed_drafts": 210, "spec_correct_drafts_histogram": [0, 0, 1, 5, 0, 1, 2, 1, 1, 1, 1, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 5, 0, 1, 2, 1, 1, 1, 1, 0, 0, 1], "request_received_ts": 1788171062.7757843, "api_server_dispatch_finish_ts": 1788171062.779086, "request_finished_ts": 1788171067.109959, "decode_throughput": 37682260.23849057, "response_sent_to_client_ts": 1788171067.1104217}} +{"prompt_id": 54, "output": " Patchy has just had 3 * 7 = <<3*7=21>>21 kittens\nSince Trixie's kittens are 12, both cats have 21 +12 =<<21+12=33>>33 kittens.\nCombining the adopted kittens with those from the two cats, the Doubtfire family now has 7 + 33 = <<7+33=40>>40 kittens.\n#### 40\n\n", "correct": true, "error": null, "completion_tokens": 88, "spec_accept_length": 5.5, "spec_accept_length_from_histogram": 5.5625, "spec_accept_rate": 0.30416666666666664, "spec_accepted_drafts": 73, "spec_proposed_drafts": 240, "spec_accept_histogram": [0, 3, 2, 2, 3, 1, 0, 1, 1, 2, 1], "spec_verify_ct": 16, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "2ec25579ebe94842a7a337bbbb515248", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 749, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171066.4627063, "queue_time": 0.00015425682067871094, "reasoning_tokens": 0, "completion_tokens": 88, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 88}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.130107218399644, "spec_accept_rate": 0.30416666666666664, "spec_accept_length": 5.5, "spec_num_correct_drafts": 73, "spec_num_proposed_drafts": 240, "spec_verify_ct": 16, "spec_accepted_drafts": 73, "spec_proposed_drafts": 240, "spec_correct_drafts_histogram": [0, 3, 2, 2, 3, 1, 0, 1, 1, 2, 1], "spec_accept_histogram": [0, 3, 2, 2, 3, 1, 0, 1, 1, 2, 1], "request_received_ts": 1788171063.9261012, "api_server_dispatch_finish_ts": 1788171063.929208, "request_finished_ts": 1788171069.0562084, "decode_throughput": 12615197.66212019, "response_sent_to_client_ts": 1788171069.057062}} +{"prompt_id": 55, "output": " Jean has 30 - 2 = <<30-2=28>>28 lollipops\nJean can fill up 28 / 2 = <<28/2=14>>14 bags with lollipops\n#### 14\n\n", "correct": true, "error": null, "completion_tokens": 49, "spec_accept_length": 6.125, "spec_accept_length_from_histogram": 6.25, "spec_accept_rate": 0.35, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_accept_histogram": [0, 0, 3, 1, 1, 0, 0, 0, 0, 1, 2], "spec_verify_ct": 8, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "e9533bf1ff3f4e02a6106e9f1ff7a6f0", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 713, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171066.5961456, "queue_time": 0.0001575946807861328, "reasoning_tokens": 0, "completion_tokens": 49, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 49}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 3.9816475389525294, "spec_accept_rate": 0.35, "spec_accept_length": 6.125, "spec_num_correct_drafts": 42, "spec_num_proposed_drafts": 120, "spec_verify_ct": 8, "spec_accepted_drafts": 42, "spec_proposed_drafts": 120, "spec_correct_drafts_histogram": [0, 0, 3, 1, 1, 0, 0, 0, 0, 1, 2], "spec_accept_histogram": [0, 0, 3, 1, 1, 0, 0, 0, 0, 1, 2], "request_received_ts": 1788171063.9215264, "api_server_dispatch_finish_ts": 1788171063.9247196, "request_finished_ts": 1788171067.903174, "decode_throughput": 11672428.389083909, "response_sent_to_client_ts": 1788171067.903747}} +{"prompt_id": 56, "output": " It costs him $14 to go to the movies because 7 + 7 = <<7+7=14>>14\nHe can go to the movies 3 times because 42 / 14 = <<42/14=3>>3\n#### 3\n\n", "correct": true, "error": null, "completion_tokens": 56, "spec_accept_length": 6.222222222222222, "spec_accept_length_from_histogram": 6.333333333333333, "spec_accept_rate": 0.35555555555555557, "spec_accepted_drafts": 48, "spec_proposed_drafts": 135, "spec_accept_histogram": [0, 1, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1], "spec_verify_ct": 9, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "19ee8bce1d2e4b18b22dfe2522c2ec32", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 712, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171067.77493, "queue_time": 0.00015306472778320312, "reasoning_tokens": 0, "completion_tokens": 56, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 56}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 4.934992738999426, "spec_accept_rate": 0.35555555555555557, "spec_accept_length": 6.222222222222222, "spec_num_correct_drafts": 48, "spec_num_proposed_drafts": 135, "spec_verify_ct": 9, "spec_accepted_drafts": 48, "spec_proposed_drafts": 135, "spec_correct_drafts_histogram": [0, 1, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 1, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1], "request_received_ts": 1788171064.5249224, "api_server_dispatch_finish_ts": 1788171064.528096, "request_finished_ts": 1788171069.4599152, "decode_throughput": 13895482.428235294, "response_sent_to_client_ts": 1788171069.4603887}} +{"prompt_id": 57, "output": " The boxes can weigh up to 5000 pounds - 3755 pounds = <<5000-3755=1245>>1245 pounds in total.\nThere can be 1245 / 15 = <<1245/15=83>>83 boxes loaded onto the truck without exceeding the bridge's weight limit.\n#### 83\n\n", "correct": true, "error": null, "completion_tokens": 69, "spec_accept_length": 6.2727272727272725, "spec_accept_length_from_histogram": 6.363636363636363, "spec_accept_rate": 0.3575757575757576, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 2, 0, 0, 4, 3, 1, 0, 0, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "ba31f25880b14c168c40a5a0f5b25dfb", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 740, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171068.0578175, "queue_time": 0.00015425682067871094, "reasoning_tokens": 0, "completion_tokens": 69, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 69}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.419019355904311, "spec_accept_rate": 0.3575757575757576, "spec_accept_length": 6.2727272727272725, "spec_num_correct_drafts": 59, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 2, 0, 0, 4, 3, 1, 0, 0, 1], "spec_accept_histogram": [0, 0, 2, 0, 0, 4, 3, 1, 0, 0, 1], "request_received_ts": 1788171064.5295408, "api_server_dispatch_finish_ts": 1788171064.532471, "request_finished_ts": 1788171069.9485602, "decode_throughput": 19473114.823843177, "response_sent_to_client_ts": 1788171069.9490383}} +{"prompt_id": 58, "output": " He spent $40.00 on groceries but they charged him a 25% fee so that's 40*.25 = $10.00\nThere is also a $3.00 delivery fee and a $4.00 tip for an extra $3 + $4 = $<<3+4=7.00>>7.00 in fees\nHis groceries were $40.00, there's a $10.00 fee and another $7.00 in delivery/tip fees for a final total of $40 + $10 + $7 = $<<40+10+7=57.00>>57.00\n#### 57\n\n", "correct": true, "error": null, "completion_tokens": 132, "spec_accept_length": 5.076923076923077, "spec_accept_length_from_histogram": 5.115384615384615, "spec_accept_rate": 0.2743589743589744, "spec_accepted_drafts": 107, "spec_proposed_drafts": 390, "spec_accept_histogram": [0, 5, 5, 4, 1, 3, 2, 4, 0, 1, 0, 0, 1], "spec_verify_ct": 26, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "8dcf1b0f471a4219b7c7121db86e8b6f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 747, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171068.538077, "queue_time": 0.0001659393310546875, "reasoning_tokens": 0, "completion_tokens": 132, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 132}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 8.250277529936284, "spec_accept_rate": 0.2743589743589744, "spec_accept_length": 5.076923076923077, "spec_num_correct_drafts": 107, "spec_num_proposed_drafts": 390, "spec_verify_ct": 26, "spec_accepted_drafts": 107, "spec_proposed_drafts": 390, "spec_correct_drafts_histogram": [0, 5, 5, 4, 1, 3, 2, 4, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 5, 5, 4, 1, 3, 2, 4, 0, 1, 0, 0, 1], "request_received_ts": 1788171065.4539702, "api_server_dispatch_finish_ts": 1788171065.4572413, "request_finished_ts": 1788171073.7042477, "decode_throughput": 44.83700248453688, "response_sent_to_client_ts": 1788171073.7047384}} +{"prompt_id": 59, "output": " First find the number of fruit in clusters: 20 fruit/cluster * 6 clusters = <<20*6=120>>120 fruit\nThen add the number of individual fruit to find the total number of fruit: 120 fruit + 67 fruit = <<120+67=187>>187 fruit\n#### 187\n\n", "correct": true, "error": null, "completion_tokens": 66, "spec_accept_length": 5.076923076923077, "spec_accept_length_from_histogram": 5.153846153846154, "spec_accept_rate": 0.27692307692307694, "spec_accepted_drafts": 54, "spec_proposed_drafts": 195, "spec_accept_histogram": [0, 2, 1, 4, 1, 3, 0, 0, 1, 0, 0, 1], "spec_verify_ct": 13, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "3174004539a74cfe922578e48c3051a6", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 697, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171068.873423, "queue_time": 0.00012636184692382812, "reasoning_tokens": 0, "completion_tokens": 66, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 66}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.64719158783555, "spec_accept_rate": 0.27692307692307694, "spec_accept_length": 5.076923076923077, "spec_num_correct_drafts": 54, "spec_num_proposed_drafts": 195, "spec_verify_ct": 13, "spec_accepted_drafts": 54, "spec_proposed_drafts": 195, "spec_correct_drafts_histogram": [0, 2, 1, 4, 1, 3, 0, 0, 1, 0, 0, 1], "spec_accept_histogram": [0, 2, 1, 4, 1, 3, 0, 0, 1, 0, 0, 1], "request_received_ts": 1788171065.8073256, "api_server_dispatch_finish_ts": 1788171065.810489, "request_finished_ts": 1788171071.4545171, "decode_throughput": 145.5518742832151, "response_sent_to_client_ts": 1788171071.455095}} +{"prompt_id": 60, "output": " 20% of 25 oranges is (20/100)*25 = <<(20/100)*25=5>>5 oranges\nThe total number of bad, unripe and sour oranges is therefore 1+5+2 = <<1+5+2=8>>8 oranges\nThe rest which is 25-8 = <<25-8=17>>17 oranges are good\n#### 17\n\n", "correct": true, "error": null, "completion_tokens": 86, "spec_accept_length": 5.733333333333333, "spec_accept_length_from_histogram": 5.866666666666666, "spec_accept_rate": 0.3244444444444444, "spec_accepted_drafts": 73, "spec_proposed_drafts": 225, "spec_accept_histogram": [0, 2, 2, 2, 2, 3, 0, 0, 1, 2, 0, 0, 1], "spec_verify_ct": 15, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "c20ad8afbdbc4950b349fd2c5a9a3d6d", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 704, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171070.0984845, "queue_time": 0.00012946128845214844, "reasoning_tokens": 0, "completion_tokens": 86, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 86}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 6.288965605199337, "spec_accept_rate": 0.3244444444444444, "spec_accept_length": 5.733333333333333, "spec_num_correct_drafts": 73, "spec_num_proposed_drafts": 225, "spec_verify_ct": 15, "spec_accepted_drafts": 73, "spec_proposed_drafts": 225, "spec_correct_drafts_histogram": [0, 2, 2, 2, 2, 3, 0, 0, 1, 2, 0, 0, 1], "spec_accept_histogram": [0, 2, 2, 2, 2, 3, 0, 0, 1, 2, 0, 0, 1], "request_received_ts": 1788171066.930689, "api_server_dispatch_finish_ts": 1788171066.9339535, "request_finished_ts": 1788171073.2196548, "decode_throughput": 25633493.90254178, "response_sent_to_client_ts": 1788171073.2200944}} +{"prompt_id": 61, "output": " The total cost of the broch was 500+800=$<<500+800=1300>>1300\nThe insurance was 1300*.1=$<<1300*.1=130>>130\nSo she pays 1300+130=$<<1300+130=1430>>1430\n#### 1430\n\n", "correct": true, "error": null, "completion_tokens": 69, "spec_accept_length": 6.2727272727272725, "spec_accept_length_from_histogram": 6.363636363636363, "spec_accept_rate": 0.3575757575757576, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_accept_histogram": [0, 0, 1, 1, 2, 3, 2, 1, 0, 0, 0, 0, 1], "spec_verify_ct": 11, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "1fced9f4ebbd44cfba5ac345da9ddd98", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 724, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171070.2662716, "queue_time": 0.00012826919555664062, "reasoning_tokens": 0, "completion_tokens": 69, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 69}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 5.699625255074352, "spec_accept_rate": 0.3575757575757576, "spec_accept_length": 6.2727272727272725, "spec_num_correct_drafts": 59, "spec_num_proposed_drafts": 165, "spec_verify_ct": 11, "spec_accepted_drafts": 59, "spec_proposed_drafts": 165, "spec_correct_drafts_histogram": [0, 0, 1, 1, 2, 3, 2, 1, 0, 0, 0, 0, 1], "spec_accept_histogram": [0, 0, 1, 1, 2, 3, 2, 1, 0, 0, 0, 0, 1], "request_received_ts": 1788171067.1207526, "api_server_dispatch_finish_ts": 1788171067.1240063, "request_finished_ts": 1788171072.8203778, "decode_throughput": 19946576.705914494, "response_sent_to_client_ts": 1788171072.820937}} +{"prompt_id": 62, "output": " First find how many years Marcy works after 20 years: 30 years - 20 years = <<30-20=10>>10 years\nThen multiply that number by the amount of her pension she gets per year: 10 years * 5% = 50%\nThen multiply that percentage by the total value of the pension to find how much she gets: $50,000 * 50% = $<<50000*50*.01=25000>>25,000\n#### 25000\n\n", "correct": true, "error": null, "completion_tokens": 106, "spec_accept_length": 4.416666666666667, "spec_accept_length_from_histogram": 4.458333333333334, "spec_accept_rate": 0.23055555555555557, "spec_accepted_drafts": 83, "spec_proposed_drafts": 360, "spec_accept_histogram": [1, 3, 4, 5, 5, 3, 0, 2, 1], "spec_verify_ct": 24, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "b45c310e65904dadbc38a55907b94f8f", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 731, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171070.6017368, "queue_time": 0.00012564659118652344, "reasoning_tokens": 0, "completion_tokens": 106, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 106}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.041785742621869, "spec_accept_rate": 0.23055555555555557, "spec_accept_length": 4.416666666666667, "spec_num_correct_drafts": 83, "spec_num_proposed_drafts": 360, "spec_verify_ct": 24, "spec_accepted_drafts": 83, "spec_proposed_drafts": 360, "spec_correct_drafts_histogram": [1, 3, 4, 5, 5, 3, 0, 2, 1], "spec_accept_histogram": [1, 3, 4, 5, 5, 3, 0, 2, 1], "request_received_ts": 1788171067.9173245, "api_server_dispatch_finish_ts": 1788171067.9230635, "request_finished_ts": 1788171074.9591103, "decode_throughput": 24305894.474506844, "response_sent_to_client_ts": 1788171074.9597068}} +{"prompt_id": 63, "output": " A year has twelve months, so a half of the year is 12/2 = <<12/2=6>>6 months,\nFor the first six months, the streaming service charged her a total of 6*140 = $<<6*140=840>>840\nFor the next half of the year, the streaming service company charged her 10/100*140 = $<<10/100*140=14>>14 less\nShe paid $140-$14 = $<<140-14=126>>126 each month for the remaining half of the year,\nIn total, she paid 126*6 = $<<126*6=756>>756 for the other half of the year for the streaming service.\nThe total amount of money she paid at the end of the year is $840+$756 = $<<840+756=1596>>1596\n#### 1596\n\n", "correct": true, "error": null, "completion_tokens": 185, "spec_accept_length": 5.606060606060606, "spec_accept_length_from_histogram": 5.636363636363637, "spec_accept_rate": 0.3090909090909091, "spec_accepted_drafts": 153, "spec_proposed_drafts": 495, "spec_accept_histogram": [0, 4, 7, 5, 0, 5, 2, 2, 5, 1, 2], "spec_verify_ct": 33, "finish_reason": {"type": "stop", "matched": "Question"}, "meta_info": {"id": "9e1861ca22984ac48fba15cc0ae91ba8", "finish_reason": {"type": "stop", "matched": "Question"}, "prompt_tokens": 734, "weight_version": "default", "num_retractions": 0, "forward_entry_time": 1788171071.670827, "queue_time": 0.00013375282287597656, "reasoning_tokens": 0, "completion_tokens": 185, "cached_tokens": 0, "weight_versions": [{"version": "default", "start": 0, "end": 185}], "cached_tokens_details": null, "dp_rank": null, "e2e_latency": 7.202895070891827, "spec_accept_rate": 0.3090909090909091, "spec_accept_length": 5.606060606060606, "spec_num_correct_drafts": 153, "spec_num_proposed_drafts": 495, "spec_verify_ct": 33, "spec_accepted_drafts": 153, "spec_proposed_drafts": 495, "spec_correct_drafts_histogram": [0, 4, 7, 5, 0, 5, 2, 2, 5, 1, 2], "spec_accept_histogram": [0, 4, 7, 5, 0, 5, 2, 2, 5, 1, 2], "request_received_ts": 1788171069.0688148, "api_server_dispatch_finish_ts": 1788171069.0722227, "request_finished_ts": 1788171076.27171, "decode_throughput": 140.2172170198619, "response_sent_to_client_ts": 1788171076.2722104}} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c8.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c8.log new file mode 100644 index 0000000..74ddefd --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/bench/gsm8k_c8.log @@ -0,0 +1,28 @@ +gsm8k_c8 174.1.60.1: docker run --rm --network host -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/../../../datasets/gsm8k/test.jsonl:/dataset/gsm8k.jsonl:ro -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/bench_gsm8k_acceptance.py:/bench.py:ro -v /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740:/results --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 /bench.py --data-path /dataset/gsm8k.jsonl --num-questions 64 --num-shots 5 --max-new-tokens 512 --temperature 0 --top-p 1 --parallel 8 --speculative-algorithm DFLASH --host 174.1.60.1 --port 31000 --backend srt --output-file /results/bench/gsm8k_c8.jsonl --summary-file /results/bench/gsm8k_c8_summary.json +/opt/kimi-dflash/python/sglang/srt/layers/quantization/awq/awq.py:53: UserWarning: Only CUDA, HIP and XPU support AWQ currently. + warnings.warn(f"Only CUDA, HIP and XPU support AWQ currently.") +/opt/kimi-dflash/python/sglang/srt/layers/quantization/gguf.py:69: UserWarning: Only CUDA, MUSA and NPU support GGUF quantization currently. + warnings.warn(f"Only CUDA, MUSA and NPU support GGUF quantization currently.") + 0%| | 0/64 [00:00 [options] + warnings.warn( +[2026-08-31 09:46:22] KV cache is forced as chunk cache for decode server +[2026-08-31 09:46:22] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:46:22] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:46:25] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:46:25] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.86, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.5:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 0, 'tp_size': 32, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 176937127, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.21, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': None, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'decode', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': 16, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:46:25] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:25] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:25] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:25] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:25] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:31] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:31] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:32] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:32] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:46:32] No HuggingFace chat template found +[2026-08-31 09:46:32] No chat template found, defaulting to 'string' content format +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:51 TP5 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:51 TP5 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:56] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:56] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:56 TP5 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:56 TP5 EP0] Multimodal data loading enabled with 16 worker threads (auto). +/opt/kimi-dflash/python/sglang/srt/utils/hf_transformers/tokenizer.py:433: UserWarning: Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead. + warnings.warn( +[2026-08-31 09:46:56] Applying special tokens cache patch for Kimi tokenizer: +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:56.508476 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:56.508572 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:56.508594 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:46:56.508631 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15089 +I0831 09:46:56.508741 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +I0831 09:46:56.514320 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:56.519191 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:56.523962 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:56.529505 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:56.531534 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:56.531612 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:56.531656 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.537504 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:56.538760 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:46:56.538784 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.562997 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:56.564376 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:46:56.564402 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.590972 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:56.592375 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:46:56.592399 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.618980 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:56.620348 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:46:56.620483 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:56 TP5 EP0] Init torch distributed begin. +[2026-08-31 09:46:56 TP6 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:56 TP6 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:57 TP3 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:57 TP3 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:57 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:57 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:58 TP7 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:58 TP7 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:58 TP1 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:58 TP1 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:59 TP4 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:59 TP4 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:47:00 TP6 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:00 TP6 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:47:00 TP2 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:47:00 TP2 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:00.412621 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:00.412681 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:00.412690 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:00.412717 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15305 +I0831 09:47:00.412791 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:00.436543 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:00.443938 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:00.451467 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:00.460381 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:00.464072 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:00.464169 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:00.464226 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.491043 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:00.492483 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:00.492514 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.518980 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:00.520355 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:00.520380 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.546947 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:00.548306 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:00.548331 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.574934 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:00.576288 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:00.576414 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:00 TP6 EP0] Init torch distributed begin. +[2026-08-31 09:47:00 TP3 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:00 TP3 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:01.008545 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:01.008625 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:01.008637 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:01.008670 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16829 +I0831 09:47:01.008787 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:01.013367 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:01.038249 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:01.044988 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:01.051884 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:01.054628 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:01.054724 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:01.054777 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.061849 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:01.063266 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:01.063295 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.086905 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:01.088263 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:01.088289 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.114934 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:01.116294 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:01.116317 231 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:47:01 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:01 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +I0831 09:47:01.142899 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:01.144273 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:01.144394 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:01 TP3 EP0] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:01.312850 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:01.312904 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:01.312912 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:01.312937 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15756 +I0831 09:47:01.313014 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:01.336449 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:01.340915 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:01.346061 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:01.351586 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:01.353834 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:01.353921 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:01.353972 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.360467 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:01.361785 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:01.361812 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.386942 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:01.388332 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:01.388362 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.414891 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:01.416260 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:01.416285 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.442946 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:01.444378 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:01.444506 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:01 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:47:01 TP7 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:01 TP7 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:01.654273 235 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:01.654327 235 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:01.654336 235 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:01.654357 235 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16693 +I0831 09:47:01.654423 235 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:01.675889 235 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:01.702773 235 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:01.709753 235 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:01.716742 235 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:01.719381 235 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:01.719475 235 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:01.719529 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.726668 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:01.728080 235 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:01.728107 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.754936 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:01.756278 235 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:01.756309 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.782934 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:01.784304 235 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:01.784332 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:01.791632 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:01.792995 235 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:01.793134 235 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:01 TP7 EP0] Init torch distributed begin. +[2026-08-31 09:47:02 TP1 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:02 TP1 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:02.695990 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:02.696061 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:02.696075 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:02.696103 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15263 +I0831 09:47:02.696202 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:02.722229 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:02.728830 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:02.735767 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:02.742777 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:02.745424 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:02.745515 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:02.745569 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:02.752698 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:02.754125 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:02.754195 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:02.778932 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:02.780292 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:02.780324 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:02.806954 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:02.808336 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:02.808364 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:02.834970 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:02.836316 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:02.836444 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:02 TP1 EP0] Init torch distributed begin. +[2026-08-31 09:47:03 TP4 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:03 TP4 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:03.235311 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:03.235359 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:03.235366 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:03.235386 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15594 +I0831 09:47:03.235450 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:03.255211 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:03.258509 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:03.261724 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:03.265254 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:03.266417 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:03.266462 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:03.266485 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:03.270087 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:03.270907 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:03.270920 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:03.293193 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:03.294323 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:03.294348 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:03.318902 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:03.320277 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:03.320300 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:03.346930 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:03.348354 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:03.348491 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:03 TP4 EP0] Init torch distributed begin. +[2026-08-31 09:47:04 TP2 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:47:04 TP2 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:47:05.006716 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:47:05.006794 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:47:05.006805 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0831 09:47:05.006834 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15909 +I0831 09:47:05.006916 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:47:05.033680 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:47:05.040295 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:47:05.047327 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:05.054328 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:05.057032 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:05.057137 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:05.057200 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:05.083019 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:05.084457 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0831 09:47:05.084491 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:05.110929 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:05.112294 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0831 09:47:05.112323 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:05.139001 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:05.140453 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0831 09:47:05.140483 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:05.167032 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:05.168421 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0831 09:47:05.168577 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:05 TP2 EP0] Init torch distributed begin. +[2026-08-31 09:47:08 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:47:09 TP1 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP0 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP2 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP4 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP7 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP3 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP6 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP5 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP0 EP0] Init torch distributed ends. elapsed=8.43 s, mem usage=0.45 GB +[2026-08-31 09:47:09 TP5 EP0] Init torch distributed ends. elapsed=13.25 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP6 EP0] Init torch distributed ends. elapsed=9.30 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP7 EP0] Init torch distributed ends. elapsed=8.08 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP3 EP0] Init torch distributed ends. elapsed=8.73 s, mem usage=0.45 GB +[2026-08-31 09:47:09 TP1 EP0] Init torch distributed ends. elapsed=7.04 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP4 EP0] Init torch distributed ends. elapsed=6.53 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP2 EP0] Init torch distributed ends. elapsed=4.71 s, mem usage=0.48 GB +[2026-08-31 09:47:10 TP3 EP0] Load weight begin. avail mem=82.13 GB +[2026-08-31 09:47:10 TP3 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP3 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP3 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP5 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP0 EP0] Load weight begin. avail mem=82.13 GB +[2026-08-31 09:47:10 TP5 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP1 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP4 EP0] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP6 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP1 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP4 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP6 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP5 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP5 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP1 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP4 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP1 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP4 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP6 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP6 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP7 EP0] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP2 EP0] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP7 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP2 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP7 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP2 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP7 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP2 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:12 TP0 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP1 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP2 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP3 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP4 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP6 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP5 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP7 EP0] multimem all-gather disabled because the TP group spans across nodes. + Multi-thread loading shards: 0% Completed | 0/96 [00:00, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:51:23 TP0 EP0] DFLASH fused KV materialization enabled. n_layers=6, num_kv_heads=1, head_dim=128 +[2026-08-31 09:51:24 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP7 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP6 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP2 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP5 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP4 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP3 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP1 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP2 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP0 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP2 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP6 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP4 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP7 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP5 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP3 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP1 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP5 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP6 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP1 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP3 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP4 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP7 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP2 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP2 EP0] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP0 EP0] Memory pool end. avail mem=9.20 GB +[2026-08-31 09:51:24 TP1 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP1 EP0] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP6 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP6 EP0] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP5 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP7 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP3 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP4 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP7 EP0] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP5 EP0] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP3 EP0] Memory pool end. avail mem=9.20 GB +[2026-08-31 09:51:24 TP4 EP0] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:27 TP2 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP7 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP7 EP0] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP2 EP0] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP4 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP4 EP0] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP0 EP0] Memory pool end. avail mem=8.36 GB +[2026-08-31 09:51:27 TP3 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP3 EP0] Memory pool end. avail mem=8.36 GB +[2026-08-31 09:51:27 TP6 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP6 EP0] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP1 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP1 EP0] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP5 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP5 EP0] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP7 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP4 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP2 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:51:27 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:51:27 TP0 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP3 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP5 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP1 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP6 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp0_pp0_dp0.json +[2026-08-31 09:51:27 TP3 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp3_pp0_dp0.json +[2026-08-31 09:51:27 TP4 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp4_pp0_dp0.json +[2026-08-31 09:51:27 TP5 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp5_pp0_dp0.json +[2026-08-31 09:51:27 TP7 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp7_pp0_dp0.json +[2026-08-31 09:51:27 TP1 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp1_pp0_dp0.json +[2026-08-31 09:51:27 TP6 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp6_pp0_dp0.json +[2026-08-31 09:51:27 TP2 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp2_pp0_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/8 [00:000, gen1: 1885->0, gen2: 1109557->0 +[2026-08-31 09:55:11 TP1 EP0] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109545->0 +[2026-08-31 09:55:11 TP4 EP0] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109553->0 +[2026-08-31 09:55:11 TP0 EP0] Freezing GC in Scheduler process. gen0: 414->0, gen1: 2699->0, gen2: 1109868->0 +[2026-08-31 09:55:11 TP2 EP0] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109547->0 +[2026-08-31 09:55:11 TP5 EP0] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109546->0 +[2026-08-31 09:55:11] Freezing GC in Tokenizer Manager process. gen0: 919->0, gen1: 3330->0, gen2: 1051166->0 +[2026-08-31 09:55:11] The server is fired up and ready to roll! +[2026-08-31 09:55:11 TP6 EP0] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109548->0 +[2026-08-31 09:55:11 TP3 EP0] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109552->0 +[2026-08-31 09:55:11] Freezing GC in Detokenizer Manager process. gen0: 780->0, gen1: 3129->0, gen2: 869158->0 +I0831 09:55:20.385135 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.393936 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.394201 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.394454 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.433565 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.433820 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.434111 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.434365 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.443164 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.446086 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.457504 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.482111 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.490996 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.499799 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.500089 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.500365 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.500633 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.500977 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.501242 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.501464 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.501698 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.505136 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.511446 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.511672 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.535360 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.538548 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.541625 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.541851 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.542086 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.542328 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.542593 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.542843 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.543073 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.543267 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.543493 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.549505 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.552603 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.578584 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.578884 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.590474 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.593645 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.593894 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.594199 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.594472 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.594729 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.594960 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.595238 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.595498 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.595814 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.607172 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.607395 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.607673 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.607934 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.610703 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.612025 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.619789 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.620287 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.621881 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.635748 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.635970 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.636209 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.637362 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.643389 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.644788 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.647404 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.647634 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.647909 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.648190 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.649222 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.656039 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.656263 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.656509 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.656766 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.664623 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.666234 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.667567 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.667891 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668154 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668340 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668640 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668896 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.669173 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.669446 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.669689 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.669978 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.674093 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.675714 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.680338 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.680579 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.680878 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.686065 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.686306 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.686568 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.686805 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.688877 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.689122 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.689296 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691989 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.692222 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.692559 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.692785 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.694262 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.694532 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.694780 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.696043 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.698016 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.698246 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.699313 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.699561 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.702633 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.703534 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.704682 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.709425 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.709708 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.709946 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.710215 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.710479 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.710713 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.710948 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.711364 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715520 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.719862 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.720649 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.722096 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.730062 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731564 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.738564 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.744866 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.745075 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.745306 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.747440 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.747728 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.749074 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.752035 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.753399 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.759246 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.760195 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.760403 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.770192 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.770453 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.770649 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.770895 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.771132 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.775832 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.776489 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.776738 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.776978 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.777184 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.777405 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.785292 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.786753 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.793282 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.793483 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.798766 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.798947 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.799144 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.802593 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.805173 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.808871 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.809080 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.809306 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.809517 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.809811 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.809988 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.810158 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.810370 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.811451 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819967 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820209 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820478 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820744 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820966 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.821197 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.827224 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.828711 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.836210 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.836444 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.836547 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.836730 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.836932 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.837927 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.850889 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.851082 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.851305 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.854967 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857290 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857560 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857792 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857967 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858031 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858264 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858487 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.859288 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.864825 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.868453 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.868702 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.868917 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.869128 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.869355 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.869606 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.869814 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.870026 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.872857 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.877142 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.878352 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.881127 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.882647 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.884990 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.887358 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.887579 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.887795 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.887992 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.890529 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.892035 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.904485 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.904808 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.904939 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.905051 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.905206 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.905387 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.906212 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908043 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908237 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908455 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908663 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908846 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.909042 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.909269 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.909546 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.909754 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.909960 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.910161 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.910384 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.910647 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.910881 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.911077 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.915119 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.915366 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.916572 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.919157 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.921211 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928586 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928778 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929057 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929268 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929564 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929740 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929961 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.934556 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.934805 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935066 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937450 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938110 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938361 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.939531 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.941953 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.943414 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.951691 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953155 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.959333 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.966523 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.966753 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.966976 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.967885 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.974200 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.975545 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.979921 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982240 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982456 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982685 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982913 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983153 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983395 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983637 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.993759 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.994019 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.997896 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.002003 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.003387 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009033 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009315 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009672 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009763 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.010020 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.012859 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013073 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013296 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013527 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013722 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013988 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.019148 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.023824 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.024041 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.024250 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.028048 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.029206 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.029415 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.032050 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.032250 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.032596 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.032850 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.033682 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.036482 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.038228 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.043747 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.043983 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.050789 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.052047 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.057186 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.057837 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.058055 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.058466 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.059248 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.063107 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.064309 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.068130 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.068387 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.068608 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.068842 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.069047 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.069272 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.069481 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.069694 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.070490 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.070681 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.070703 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.070899 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.071774 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.080799 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.081024 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.081862 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.082042 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.082195 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.082242 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.083176 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.089670 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.089936 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.090103 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.090283 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.090451 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.099225 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.099478 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.109663 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.109822 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.109910 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110037 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110126 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110253 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110482 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110724 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110838 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.110915 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.111141 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.113976 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.116101 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.117246 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.121361 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.121557 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.121807 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.122004 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.122270 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.122500 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.122700 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.122928 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.125197 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.129302 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.129984 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130196 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130366 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130631 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.131251 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132718 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.135440 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.135658 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.135867 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136121 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136298 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136531 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136698 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136762 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136984 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136996 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.138283 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.145439 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.145721 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.146814 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.154562 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.154863 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157827 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158082 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158324 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158571 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.173578 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.173821 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.174254 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.174439 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.174650 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.174891 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.175110 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.175642 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.181558 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.182945 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.186182 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.186445 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189155 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189373 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189611 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189733 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189787 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.190044 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.193775 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195034 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195056 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196527 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196768 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196978 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.197926 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.200170 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.200399 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.201583 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.203384 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.203644 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.207980 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.208236 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.208503 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.208985 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.209139 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.211371 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.211655 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.219911 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.221532 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.221807 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.231874 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232087 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232277 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232519 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232770 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232767 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232985 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.233264 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.233456 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.234046 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.245054 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.245342 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.245539 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.246311 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250303 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250581 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250761 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.253377 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.258428 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.258618 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.258785 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.259625 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.259735 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.263803 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264837 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.265060 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.266112 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.266918 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.267138 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.267357 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.267573 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.267787 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.268038 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.268285 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.276016 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.276228 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.276418 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.276690 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.276929 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277539 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.279641 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.287506 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.287691 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.287914 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.290330 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.291821 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.293008 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295169 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295370 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295593 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295769 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295815 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.296000 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.296028 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.296330 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.297737 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.298695 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.301421 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.301647 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.301909 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.307155 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.307395 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.310029 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.310245 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.311439 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.312199 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.312489 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.312703 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.312897 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.313103 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.313496 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.320261 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.325500 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.325703 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.325901 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.326534 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.331472 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.334101 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.334297 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.334295 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.336915 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.338205 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.360706 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.360949 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.361156 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.361327 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.361534 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.361766 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.361986 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.362226 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.362484 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.362702 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.362913 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.381932 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.382203 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.382418 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.382653 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.382838 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383070 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383330 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383538 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.397185 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.398784 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.401818 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.402081 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.402251 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.402441 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.402655 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.402858 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.404932 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.405665 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.411243 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.422935 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423134 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423312 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.426625 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.429343 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.430629 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.432538 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.442487 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.442693 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.442891 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.443121 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.443339 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.443567 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.443771 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.443987 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.444218 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.444444 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.444655 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.444897 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.445091 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.445344 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.445473 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.446902 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.448012 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.452663 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.458696 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.458851 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.460083 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.465596 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.467705 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.467947 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.468165 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.468375 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.468585 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.468801 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.468976 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.469197 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.469403 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.470454 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.470665 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.470898 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.474224 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.476832 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.477161 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.477406 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.478559 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.479387 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.481904 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.482116 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.483279 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.498195 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.499646 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.505169 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.505848 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506211 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.511003 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.511188 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.511428 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.511441 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.511641 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.519007 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.521554 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.523131 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.523362 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.523532 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.524514 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.524727 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.527002 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.527544 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.527752 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.527935 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.529824 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.530633 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.531071 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.533177 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.535804 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.538407 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.541882 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.543260 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.548456 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.548753 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.554644 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.555800 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.555991 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.556205 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.556420 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.565847 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.566033 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.566268 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.566414 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.566704 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.569298 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.569377 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574608 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574635 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574833 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.575084 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.575333 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.577310 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.577527 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.578011 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.578701 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.580490 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.580717 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.590592 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.590804 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.592549 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.593437 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.593649 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.593840 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.593961 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.594050 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.599047 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.599664 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.601588 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.601788 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.602922 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.605515 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616721 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616921 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.617199 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.620092 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.622756 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.624039 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.633735 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.633992 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.634183 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.634387 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.634604 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.635231 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.642907 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.643141 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.644318 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652143 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652351 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.653362 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.683072 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.683290 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.683495 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.683717 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.683952 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.684159 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.684368 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.684581 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.684792 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.685066 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.685298 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.685534 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.706943 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.707159 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.707388 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.707681 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.707876 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.708151 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.708352 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.708587 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.708781 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.716274 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.716555 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.717554 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.735903 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.736210 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.747992 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.748225 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.748440 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.748638 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.751641 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754177 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.756717 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.759187 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.761735 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.771673 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.778991 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.779294 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.779479 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.779701 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789650 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789855 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.790103 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.797473 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.797696 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.797948 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.798169 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.803208 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.803422 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.803632 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.810834 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.811049 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.815825 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.816044 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.816238 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.821099 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.821305 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.823939 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.824214 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.825280 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 09:55:25] INFO: 174.1.60.1:49194 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 09:55:36] INFO: 174.1.60.1:46326 - "GET /health HTTP/1.1" 200 OK +[2026-08-31 09:55:25 TP0 EP0] Decode batch, #running-req: 1, #full token: 192, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 3.65, accept rate: 0.18, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 6.89, #queue-req: 0 +[2026-08-31 10:05:09] INFO: 174.1.60.1:58638 - "GET /health HTTP/1.1" 200 OK +I0831 10:06:32.025153 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.025547 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.026625 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.038115 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.038452 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.039916 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.043712 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.044075 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.045207 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.045513 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.053032 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.054387 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.055864 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.057281 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.060073 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.060356 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.060662 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.061436 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.063360 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.072307 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.072949 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.073210 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.073680 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.074529 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.074777 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.075085 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.076112 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.082175 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.083666 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.084745 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.085079 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.086153 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.087384 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.088874 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093374 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093582 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.094934 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.095937 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.096264 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.097342 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.105053 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.105650 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.105938 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.106554 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.107227 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.110997 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.111240 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.111462 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.111655 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.112397 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119129 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119414 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119657 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.120666 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.123026 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.123292 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.123520 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124380 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124922 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.126366 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.126616 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.128612 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.132350 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.132599 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.133347 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.133579 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.133764 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.133811 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.134732 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.136893 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137434 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137732 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.139053 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.142159 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.142374 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143469 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143476 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.148188 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.148459 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.148646 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.149634 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.150883 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.153906 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.160650 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.160867 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.162024 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.163350 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.163586 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.164481 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.164702 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.164858 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170240 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170509 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.171694 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.175747 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.175997 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.176036 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.177001 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.182144 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.183952 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.187474 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.188709 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:32] INFO: 174.1.60.1:58174 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:33.926203 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.927737 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.931989 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.938521 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.940084 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.944084 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.944355 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.945570 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.949950 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.953853 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.955224 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.960949 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.961239 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.962786 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.968405 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.968856 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:34] INFO: 174.1.60.1:60552 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:35.758579 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.770097 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.771520 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.772037 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.782291 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.783783 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.787595 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.789134 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.804298 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.804587 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.804852 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.805783 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.813458 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.815124 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.817991 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.818236 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.819496 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.839099 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.839346 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.839579 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.839790 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.840008 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.840539 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.845659 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.853612 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.853933 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.854173 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.855252 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.859723 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:37] INFO: 174.1.60.1:60558 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:38] INFO: 174.1.60.1:60572 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:39.950922 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.980537 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.991514 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.997699 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.999344 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.002673 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.004179 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.009007 1501 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.013356 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.014210 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.014442 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.015807 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.025856 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.031415 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.031658 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.032768 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:40] INFO: 174.1.60.1:60576 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:42.045681 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.094672 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.126721 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:44] INFO: 174.1.60.1:60592 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:42.128132 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:43 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 12.10, accept rate: 0.74, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 0.71, #queue-req: 0 +I0831 10:06:45.615921 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.634097 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.644779 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.648653 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.663208 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.701673 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:47] INFO: 174.1.60.1:40672 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:48.472116 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.488507 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.499557 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.503995 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.514441 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.522200 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.530779 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.539918 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.549054 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.550592 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.558938 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:51] INFO: 174.1.60.1:40684 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:55] INFO: 174.1.60.1:35874 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:49 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.17, accept rate: 0.34, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 36.81, #queue-req: 0 +[2026-08-31 10:06:55 TP0 EP0] Decode batch, #running-req: 1, #full token: 960, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.85, accept rate: 0.39, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 50.62, #queue-req: 0 +I0831 10:06:56.458945 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.484690 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.495128 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.506563 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.514359 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:06:59] INFO: 174.1.60.1:35888 - "POST /generate HTTP/1.1" 200 OK +I0831 10:06:56.524739 1479 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.380396 1618 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:02] INFO: 174.1.60.1:35904 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:05] INFO: 174.1.60.1:35210 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:02 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.60, accept rate: 0.31, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 32.82, #queue-req: 0 +I0831 10:07:04.161098 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:06.949086 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:09] INFO: 174.1.60.1:35214 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:12] INFO: 174.1.60.1:35224 - "POST /generate HTTP/1.1" 200 OK +I0831 10:07:06.968375 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:06.983387 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:07.000609 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:08 TP0 EP0] Decode batch, #running-req: 1, #full token: 896, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 7.22, accept rate: 0.41, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 43.12, #queue-req: 0 +[2026-08-31 10:07:16] INFO: 174.1.60.1:34054 - "POST /generate HTTP/1.1" 200 OK +I0831 10:07:14.073452 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:15 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.60, accept rate: 0.31, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 33.33, #queue-req: 0 +I0831 10:07:17.437095 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:17.456888 1455 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:19] INFO: 174.1.60.1:34056 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:22] INFO: 174.1.60.1:34072 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:22 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.33, accept rate: 0.35, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 37.78, #queue-req: 0 +I0831 10:07:23.988735 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:24.006058 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:26] INFO: 174.1.60.1:57898 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:29] INFO: 174.1.60.1:57902 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:34] INFO: 174.1.60.1:57910 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:28 TP0 EP0] Decode batch, #running-req: 1, #full token: 768, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.28, accept rate: 0.35, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 37.44, #queue-req: 0 +[2026-08-31 10:07:34 TP0 EP0] Decode batch, #running-req: 1, #full token: 960, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.53, accept rate: 0.30, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 41.43, #queue-req: 0 +[2026-08-31 10:07:38] INFO: 174.1.60.1:40974 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:40] INFO: 174.1.60.1:40980 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:42] INFO: 174.1.60.1:40984 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:44] INFO: 174.1.60.1:43624 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:42 TP0 EP0] Decode batch, #running-req: 1, #full token: 768, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.03, accept rate: 0.34, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 29.97, #queue-req: 0 +I0831 10:07:43.879343 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:43.897684 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:07:47] INFO: 174.1.60.1:43634 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:50] INFO: 174.1.60.1:43650 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:53] INFO: 174.1.60.1:43656 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:55] INFO: 174.1.60.1:37930 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:50 TP0 EP0] Decode batch, #running-req: 1, #full token: 896, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.38, accept rate: 0.36, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 32.16, #queue-req: 0 +[2026-08-31 10:07:58 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.65, accept rate: 0.31, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 28.18, #queue-req: 0 +[2026-08-31 10:07:58] INFO: 174.1.60.1:37938 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:00] INFO: 174.1.60.1:37944 - "POST /generate HTTP/1.1" 200 OK +I0831 10:08:01.985661 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:08:01.987316 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:08:01.991817 1529 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-08-31 10:08:04] INFO: 174.1.60.1:37960 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:07] INFO: 174.1.60.1:48948 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:09] INFO: 174.1.60.1:48962 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:12] INFO: 174.1.60.1:48970 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:06 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.28, accept rate: 0.28, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 26.40, #queue-req: 0 +[2026-08-31 10:08:14] INFO: 174.1.60.1:45924 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:17] INFO: 174.1.60.1:45940 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:20] INFO: 174.1.60.1:45954 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:14 TP0 EP0] Decode batch, #running-req: 1, #full token: 768, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.10, accept rate: 0.34, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 30.89, #queue-req: 0 +[2026-08-31 10:08:20 TP0 EP0] Decode batch, #running-req: 1, #full token: 0, full token usage: 0.00, mamba num: 0, mamba usage: 0.00, accept len: 6.40, accept rate: 0.36, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 37.87, #queue-req: 0 +[2026-08-31 10:08:23] INFO: 174.1.60.1:45968 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:26] INFO: 174.1.60.1:47052 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:31] INFO: 174.1.60.1:47068 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:28 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.15, accept rate: 0.34, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 30.94, #queue-req: 0 +[2026-08-31 10:08:33] INFO: 174.1.60.1:50658 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:36] INFO: 174.1.60.1:50660 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:39] INFO: 174.1.60.1:50662 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:43] INFO: 174.1.60.1:50664 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:35 TP0 EP0] Decode batch, #running-req: 1, #full token: 896, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.83, accept rate: 0.39, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 40.10, #queue-req: 0 +[2026-08-31 10:08:42 TP0 EP0] Decode batch, #running-req: 1, #full token: 896, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.58, accept rate: 0.37, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 39.41, #queue-req: 0 +[2026-08-31 10:08:47] INFO: 174.1.60.1:53244 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:51] INFO: 174.1.60.1:53260 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:54] INFO: 174.1.60.1:47246 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:58] INFO: 174.1.60.1:47256 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:48 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.95, accept rate: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 35.54, #queue-req: 0 +[2026-08-31 10:08:54 TP0 EP0] Decode batch, #running-req: 1, #full token: 896, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.45, accept rate: 0.30, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 40.47, #queue-req: 0 +[2026-08-31 10:09:01] INFO: 174.1.60.1:47270 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:04] INFO: 174.1.60.1:39382 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:01 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.53, accept rate: 0.30, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 33.10, #queue-req: 0 +[2026-08-31 10:09:06] INFO: 174.1.60.1:39388 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:09] INFO: 174.1.60.1:39398 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:08 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.00, accept rate: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 30.17, #queue-req: 0 +[2026-08-31 10:09:11] INFO: 174.1.60.1:39414 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:14] INFO: 174.1.60.1:58358 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:17] INFO: 174.1.60.1:58362 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:20] INFO: 174.1.60.1:58368 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:16 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 6.55, accept rate: 0.37, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 32.96, #queue-req: 0 +[2026-08-31 10:09:22] INFO: 174.1.60.1:58376 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:24] INFO: 174.1.60.1:49728 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:29] INFO: 174.1.60.1:49732 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:31] INFO: 174.1.60.1:49742 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:24 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.80, accept rate: 0.32, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 29.24, #queue-req: 0 +[2026-08-31 10:09:31 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 4.90, accept rate: 0.26, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 29.50, #queue-req: 0 +[2026-08-31 10:09:34] INFO: 174.1.60.1:54104 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:37] INFO: 174.1.60.1:54114 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:41] INFO: 174.1.60.1:54126 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:46] INFO: 174.1.60.1:47160 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:39 TP0 EP0] Decode batch, #running-req: 1, #full token: 832, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 5.35, accept rate: 0.29, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 26.68, #queue-req: 0 +[2026-08-31 10:09:44 TP0 EP0] Decode batch, #running-req: 1, #full token: 896, full token usage: 0.00, mamba num: 1, mamba usage: 0.04, accept len: 4.55, accept rate: 0.24, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 33.50, #queue-req: 0 +[2026-08-31 10:10:24] INFO: 174.1.60.1:37192 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:25] INFO: 174.1.60.1:37198 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:26] INFO: 174.1.60.1:37210 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:26] INFO: 174.1.60.1:37230 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:27] INFO: 174.1.60.1:37204 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:28] INFO: 174.1.60.1:37238 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:30] INFO: 174.1.60.1:37222 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:31] INFO: 174.1.60.1:37246 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:32] INFO: 174.1.60.1:54520 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:30 TP0 EP0] Decode batch, #running-req: 6, #full token: 6528, full token usage: 0.02, mamba num: 8, mamba usage: 0.33, accept len: 7.73, accept rate: 0.45, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 23.52, #queue-req: 0 +[2026-08-31 10:10:34] INFO: 174.1.60.1:54538 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:34] INFO: 174.1.60.1:54532 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:34] INFO: 174.1.60.1:54524 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:35] INFO: 174.1.60.1:54548 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:37] INFO: 174.1.60.1:54556 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:37] INFO: 174.1.60.1:54572 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:37] INFO: 174.1.60.1:39520 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:39] INFO: 174.1.60.1:39530 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:41] INFO: 174.1.60.1:39544 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:41] INFO: 174.1.60.1:39532 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:40 TP0 EP0] Decode batch, #running-req: 5, #full token: 6400, full token usage: 0.02, mamba num: 8, mamba usage: 0.33, accept len: 6.44, accept rate: 0.36, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 137.44, #queue-req: 0 +[2026-08-31 10:10:42] INFO: 174.1.60.1:39580 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:43] INFO: 174.1.60.1:39594 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:43] INFO: 174.1.60.1:39604 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:43] INFO: 174.1.60.1:39566 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:44] INFO: 174.1.60.1:39558 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:44] INFO: 174.1.60.1:39614 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:45] INFO: 174.1.60.1:51816 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:46] INFO: 174.1.60.1:51808 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:47] INFO: 174.1.60.1:51836 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:48] INFO: 174.1.60.1:51826 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:48] INFO: 174.1.60.1:51846 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:49] INFO: 174.1.60.1:51874 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:47 TP0 EP0] Decode batch, #running-req: 4, #full token: 5440, full token usage: 0.02, mamba num: 7, mamba usage: 0.29, accept len: 6.20, accept rate: 0.35, pre-allocated usage: 0.01, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 120.25, #queue-req: 0 +[2026-08-31 10:10:50] INFO: 174.1.60.1:51856 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:51] INFO: 174.1.60.1:51886 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:51] INFO: 174.1.60.1:51872 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:51] INFO: 174.1.60.1:51888 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:52] INFO: 174.1.60.1:51896 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:53] INFO: 174.1.60.1:51914 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:54] INFO: 174.1.60.1:51912 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:54 TP0 EP0] Decode batch, #running-req: 3, #full token: 6336, full token usage: 0.02, mamba num: 8, mamba usage: 0.33, accept len: 5.93, accept rate: 0.33, pre-allocated usage: 0.01, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 123.56, #queue-req: 0 +[2026-08-31 10:10:56] INFO: 174.1.60.1:46916 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:56] INFO: 174.1.60.1:51920 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:57] INFO: 174.1.60.1:46938 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:58] INFO: 174.1.60.1:46930 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:00] INFO: 174.1.60.1:51932 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:01] INFO: 174.1.60.1:46954 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:02] INFO: 174.1.60.1:46968 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:02] INFO: 174.1.60.1:46978 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:03] INFO: 174.1.60.1:46992 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:03] INFO: 174.1.60.1:47002 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:04] INFO: 174.1.60.1:46998 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:04] INFO: 174.1.60.1:47018 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:05] INFO: 174.1.60.1:47022 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:04 TP0 EP0] Decode batch, #running-req: 4, #full token: 4608, full token usage: 0.02, mamba num: 6, mamba usage: 0.25, accept len: 6.21, accept rate: 0.35, pre-allocated usage: 0.01, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 145.31, #queue-req: 0 +[2026-08-31 10:11:05] INFO: 174.1.60.1:39066 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:06] INFO: 174.1.60.1:39090 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:07] INFO: 174.1.60.1:39078 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:07] INFO: 174.1.60.1:39116 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:09] INFO: 174.1.60.1:39106 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:09] INFO: 174.1.60.1:39132 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:09] INFO: 174.1.60.1:39142 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:11] INFO: 174.1.60.1:39146 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:12] INFO: 174.1.60.1:39160 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:10 TP0 EP0] Decode batch, #running-req: 4, #full token: 4736, full token usage: 0.02, mamba num: 6, mamba usage: 0.25, accept len: 5.76, accept rate: 0.32, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 114.14, #queue-req: 0 +[2026-08-31 10:11:13] INFO: 174.1.60.1:39158 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:13] INFO: 174.1.60.1:39144 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:14] INFO: 174.1.60.1:39168 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:16] INFO: 174.1.60.1:39172 - "POST /generate HTTP/1.1" 200 OK diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_1.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_1.log new file mode 100644 index 0000000..9aa875c --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_1.log @@ -0,0 +1,1641 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:46:14] KV cache is forced as chunk cache for decode server +[2026-08-31 09:46:14] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:46:14] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:46:17] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:46:17] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.86, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.5:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 1, 'tp_size': 32, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 24440037, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.21, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': None, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'decode', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': 16, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:46:17] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:17] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:17] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:17] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:17] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:18] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:44 TP9 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:44 TP9 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:48 TP12 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:48 TP12 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:48 TP9 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:48 TP9 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:49.145116 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:49.145196 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:49.145207 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:49.145246 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:15313 +I0831 09:46:49.145328 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:49.169463 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:49.176043 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:49.183064 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:49.190009 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:49.192700 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:49.192804 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:49.192862 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:49.200004 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:49.201457 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:49.201486 229 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:46:49 TP10 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:49 TP10 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0831 09:46:49.226336 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:49.227739 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:49.227766 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:49.254380 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:49.255811 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:49.255836 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:49.282379 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:49.283782 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:49.283905 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:49 TP9 EP1] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:49 TP8 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:49 TP8 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:50 TP15 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP15 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:50 TP11 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP11 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:50 TP14 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP14 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:51 TP13 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:51 TP13 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:52 TP12 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:52 TP12 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:52.344465 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:52.344529 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:52.344539 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:52.344569 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:16964 +I0831 09:46:52.344637 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:52.367261 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:52.373822 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:52.380627 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:52.388044 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:52.391669 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:52.391763 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:52.391819 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:52.399856 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:52.401322 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:52.401430 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:52.426369 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:52.427729 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:52.427754 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:52.454458 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:52.455837 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:52.455859 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:52.482430 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:52.483801 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:52.483932 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:52 TP12 EP1] Init torch distributed begin. +[2026-08-31 09:46:52 TP10 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:52 TP10 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:52.917550 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:52.917611 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:52.917622 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:52.917649 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:16443 +I0831 09:46:52.917754 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:52.940971 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:52.947016 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:52.953967 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:52.960834 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:52.963660 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:52.963753 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:52.963809 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:52.970966 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:52.972395 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:52.972425 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:52.998370 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:52.999747 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:52.999773 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.007084 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:53.008466 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:53.008490 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.034428 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:53.035813 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:53.035959 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:53 TP10 EP1] Init torch distributed begin. +[2026-08-31 09:46:53 TP8 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:53 TP8 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:53.540308 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:53.540376 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:53.540386 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:53.540413 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:16683 +I0831 09:46:53.540501 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:53.565325 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:53.572386 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:53.579344 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:53.606112 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:53.608772 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:53.608867 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:53.608927 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.615934 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:53.617336 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:53.617363 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.642349 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:53.643676 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:53.643721 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.670377 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:53.671723 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:53.671751 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.698385 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:53.699738 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:53.699858 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:53 TP8 EP1] Init torch distributed begin. +[2026-08-31 09:46:54 TP15 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:54 TP15 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:54.173851 235 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:54.173900 235 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:54.173907 235 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:54.173928 235 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:15821 +I0831 09:46:54.173995 235 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:54.195161 235 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:54.199601 235 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:54.204025 235 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:54.208724 235 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:54.210714 235 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:54.210788 235 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:54.210829 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.216356 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:54.217582 235 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:54.217605 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.223731 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:54.224927 235 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:54.224951 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.250487 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:54.251856 235 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:54.251880 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.278400 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:54.279764 235 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:54.279891 235 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:54 TP15 EP1] Init torch distributed begin. +[2026-08-31 09:46:54 TP11 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:54 TP11 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:54.674135 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:54.674203 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:54.674214 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:54.674245 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:16542 +I0831 09:46:54.674332 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:54.696393 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:54.702484 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:54.709015 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:54.716013 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:54.718701 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:54.718797 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:54.718859 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.726120 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:54.727542 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:54.727572 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.754623 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:54.756048 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:54.756083 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.782401 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:54.783798 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:54.783825 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.810585 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:54.812000 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:54.812156 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:54 TP11 EP1] Init torch distributed begin. +[2026-08-31 09:46:55 TP13 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:55 TP13 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:46:55 TP14 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:55 TP14 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:55.539049 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:55.539099 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:55.539108 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:55.539126 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:15958 +I0831 09:46:55.539187 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:55.559238 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:55.563143 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:55.585140 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:55.592058 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:55.594708 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:55.594802 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:55.594854 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.601871 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:55.603309 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:55.603336 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.630384 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:55.631757 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:55.631783 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.658475 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:55.659907 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:55.659945 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.686398 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:55.687810 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:55.687937 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:55 TP13 EP1] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:55.728942 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:55.729029 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:55.729041 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.6 port: 12001 +I0831 09:46:55.729067 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.6:15956 +I0831 09:46:55.729156 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:55.752802 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:55.757967 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:55.763098 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:55.768297 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:55.770066 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:55.770119 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:55.770151 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.775379 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:55.776407 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:06 +I0831 09:46:55.776424 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.781834 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:55.782759 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:06 +I0831 09:46:55.782796 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.804008 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:55.804963 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:06 +I0831 09:46:55.804979 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.830461 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:55.831822 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:06 +I0831 09:46:55.831974 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:55 TP14 EP1] Init torch distributed begin. +[2026-08-31 09:47:09 TP8 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP10 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP11 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP9 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP12 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP14 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP15 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP13 EP1] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP15 EP1] Init torch distributed ends. elapsed=15.58 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP14 EP1] Init torch distributed ends. elapsed=14.03 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP13 EP1] Init torch distributed ends. elapsed=14.17 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP12 EP1] Init torch distributed ends. elapsed=17.38 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP11 EP1] Init torch distributed ends. elapsed=15.05 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP10 EP1] Init torch distributed ends. elapsed=16.83 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP9 EP1] Init torch distributed ends. elapsed=20.58 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP8 EP1] Init torch distributed ends. elapsed=16.16 s, mem usage=0.49 GB +[2026-08-31 09:47:10 TP9 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP15 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP11 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP9 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP13 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP15 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP8 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP11 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP13 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP8 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP14 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP10 EP1] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP12 EP1] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP14 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP10 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP12 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP15 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP9 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP11 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP15 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP9 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP11 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP13 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP8 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP14 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP13 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP12 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP10 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP8 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP14 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP10 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP12 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:12 TP8 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP9 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP10 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP11 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP12 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP13 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP14 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP15 EP1] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:51:22 TP11 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP11 EP1] Load weight end. elapsed=251.84 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP13 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP13 EP1] Load weight end. elapsed=251.86 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP14 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP14 EP1] Load weight end. elapsed=251.89 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP8 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP8 EP1] Load weight end. elapsed=251.90 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP15 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP15 EP1] Load weight end. elapsed=251.94 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP10 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP10 EP1] Load weight end. elapsed=251.93 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP12 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP12 EP1] Load weight end. elapsed=252.00 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP9 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:22 TP9 EP1] Load weight end. elapsed=252.06 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP9 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP9 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP14 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP14 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP11 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP11 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP12 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP12 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP10 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP10 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP8 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP8 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP13 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP13 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP15 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP15 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:23 TP11 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP11 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP11 EP1] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP14 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP14 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP10 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP14 EP1] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP10 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP10 EP1] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP9 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP15 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP9 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP9 EP1] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP8 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP12 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP13 EP1] Init torch distributed begin. +[2026-08-31 09:51:23 TP8 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP15 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP13 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP12 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP8 EP1] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP13 EP1] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP15 EP1] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP12 EP1] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP11 EP1] Load weight end. elapsed=0.56 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP14 EP1] Load weight end. elapsed=0.55 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP15 EP1] Load weight end. elapsed=0.55 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP12 EP1] Load weight end. elapsed=0.55 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP9 EP1] Load weight end. elapsed=0.55 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP10 EP1] Load weight end. elapsed=0.56 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP13 EP1] Load weight end. elapsed=0.55 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP8 EP1] Load weight end. elapsed=0.55 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:24 TP15 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP14 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP13 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP12 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP11 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP10 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP9 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP8 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP14 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP13 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP15 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP11 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP10 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP9 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP8 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP12 EP1] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP12 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP14 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP9 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP11 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP8 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP15 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP13 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP10 EP1] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP12 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP12 EP1] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP9 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP9 EP1] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP11 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP11 EP1] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP8 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP15 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP15 EP1] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP8 EP1] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP13 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP14 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP10 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP14 EP1] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP13 EP1] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP10 EP1] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:27 TP12 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP12 EP1] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP11 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP11 EP1] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP14 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP14 EP1] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP15 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP15 EP1] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP10 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP10 EP1] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP9 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP9 EP1] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP13 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP13 EP1] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP8 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP8 EP1] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP11 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP14 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP12 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP15 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP10 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP9 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP8 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP13 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP14 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp14_pp0_dp0.json +[2026-08-31 09:51:27 TP13 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp13_pp0_dp0.json +[2026-08-31 09:51:27 TP15 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp15_pp0_dp0.json +[2026-08-31 09:51:27 TP8 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp8_pp0_dp0.json +[2026-08-31 09:51:27 TP10 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp10_pp0_dp0.json +[2026-08-31 09:51:27 TP11 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp11_pp0_dp0.json +[2026-08-31 09:51:27 TP9 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp9_pp0_dp0.json +[2026-08-31 09:51:27 TP12 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp12_pp0_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/8 [00:000, gen1: 1887->0, gen2: 1109536->0 +[2026-08-31 09:55:11 TP14 EP1] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109527->0 +[2026-08-31 09:55:11 TP9 EP1] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109538->0 +[2026-08-31 09:55:11 TP12 EP1] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109528->0 +[2026-08-31 09:55:11 TP10 EP1] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109529->0 +[2026-08-31 09:55:11 TP15 EP1] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109530->0 +[2026-08-31 09:55:11 TP13 EP1] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109535->0 +[2026-08-31 09:55:11 TP8 EP1] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109536->0 +I0831 09:55:20.386031 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.388006 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.399258 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.404611 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.406531 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.408032 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.408289 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.430903 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.431180 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.431308 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.431560 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.432901 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.440336 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.440642 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.440852 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.441138 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.449389 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.449651 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.449926 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.450206 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.451272 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.468437 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.468791 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.469094 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.469305 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.470196 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.478632 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.478859 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.479089 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.479358 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.479590 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.485322 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.492880 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.493136 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.493372 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.501369 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.501629 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.509474 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.509693 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.509894 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.510108 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.511397 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.519728 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.519984 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.521194 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.523734 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.524046 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.527245 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.527485 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.527691 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.527959 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.528245 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.531373 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.531646 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.533013 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.537988 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.540980 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.541110 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.541321 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.542483 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.548087 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.549448 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.564759 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.566105 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.569914 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.573191 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.573442 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.573686 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.573894 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.574159 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.574379 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.574604 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.574608 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.574852 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.575045 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.575275 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.576424 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.578512 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.589864 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.590122 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.590339 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.594262 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.599972 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.603091 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.603287 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.604642 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.612900 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.621270 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.621382 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.621657 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.621868 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622124 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622326 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622578 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622637 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622943 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623239 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623534 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623728 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623982 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624176 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.630549 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.631893 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.636261 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.636495 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.636729 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.639312 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.640827 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642081 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642345 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.645346 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.646358 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.649434 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.652673 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.653156 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.655851 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.657174 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.662201 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.664359 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.664577 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.665024 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.665264 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.665458 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.665673 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.665723 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.673943 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.673990 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.674194 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.674232 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.674455 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.674732 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.674973 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.675192 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.675364 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.675510 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.675563 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.675788 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.676024 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.677762 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.679080 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.681591 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.682037 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.682271 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.682521 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.683270 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.686985 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.688268 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.700403 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.701709 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.701843 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.701900 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.702109 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.706094 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.706277 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.706584 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.708498 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.712070 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.712145 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.712330 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.712554 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.712832 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.713091 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.713312 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.713539 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.713745 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.713945 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715251 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715507 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715523 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.716472 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.716903 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.717118 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.728758 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.729018 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.729206 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.730086 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.730917 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731140 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731319 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731544 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731796 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.732358 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.732442 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.733773 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.736987 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.741680 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.743052 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.748677 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.748956 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.753549 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.753862 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.755581 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.755796 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.756487 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.756762 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.756965 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.757189 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.757761 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.760907 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.762837 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.763119 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.766040 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.766292 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.766541 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.766583 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.766777 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.767247 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.767822 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.771589 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.771629 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.771785 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.772064 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.773005 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.773016 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.774343 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.774617 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.774827 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.775099 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.777876 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.778935 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.780158 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.781615 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.785743 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.788460 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.789934 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.791122 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.794060 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.794270 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.794490 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.794713 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.794924 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.796633 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.796826 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.797758 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.797938 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.797961 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.799310 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.802959 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.803216 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.803478 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.803678 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.803886 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.804141 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.804347 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.804575 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.806569 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.806787 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.810637 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.811898 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.812506 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.812731 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.812924 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.815168 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.815364 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.815598 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816341 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816546 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816576 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819069 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819676 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820363 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820912 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.824738 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.824986 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.825179 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.825397 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.825654 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.825856 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.826061 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.832839 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.834242 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.836220 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.837008 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.837257 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.837495 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.837558 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.840087 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.843101 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.843353 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.843616 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.844453 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.845546 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.847019 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.847546 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.848096 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.848306 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.848493 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.848768 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.849848 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.851141 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.856948 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857077 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857216 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857427 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.857664 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858269 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.859354 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.859655 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.860666 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.861161 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.861423 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.861651 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.862565 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863317 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863564 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863771 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.866662 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.866921 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.870565 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.870767 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.871841 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.872587 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.872781 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.873538 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.873838 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.874105 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.874971 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.875427 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.875660 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.876811 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.877714 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.878064 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.878341 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.878576 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.878696 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.878859 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.879063 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.879246 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.880012 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.881995 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.892812 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.894222 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.895223 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.895463 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.895695 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.895957 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.896195 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.896564 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.898778 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.904964 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.905189 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.905833 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.906328 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.906699 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.916383 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.916656 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.917948 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.917995 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918272 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918483 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918685 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918896 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.919094 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.919734 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.926168 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.926393 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.927793 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929571 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929826 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930075 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930804 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.933871 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935281 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936443 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937779 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938036 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938288 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938504 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938691 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938867 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.939054 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.939363 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.944186 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.945405 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.952766 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.952960 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953155 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953372 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953441 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.954916 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.956548 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.961167 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.967041 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.967356 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.967578 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.967804 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968101 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968284 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968523 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968751 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968763 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.972041 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.979456 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.979684 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.979908 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.980654 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982493 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982702 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982916 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.982967 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983151 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983170 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983361 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983371 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983575 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983796 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983997 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.984210 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.984304 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.984411 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.985755 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.987161 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.990756 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.990945 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.992213 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.994993 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.996200 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.003736 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.003995 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.004231 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.005149 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.007506 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.007726 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.007952 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.008178 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.008411 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.008590 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.008805 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009032 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009241 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009599 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.009902 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.010107 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.010118 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.010581 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.011463 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.011871 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013626 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.016258 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.018114 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.019235 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.019445 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.019511 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.019793 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.020048 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.020224 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.020526 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.021289 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.023157 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.035101 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.035359 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.036494 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.039773 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.040028 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.040279 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.040529 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.040732 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.040946 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.041200 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.046017 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.046276 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.047483 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.048149 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.054284 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.055112 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.055413 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.055619 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.055639 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.055853 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.056484 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.063817 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.063838 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.064177 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.064432 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.064640 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.064848 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.065215 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.065487 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.065757 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.066947 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.067411 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.068555 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.072479 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.073190 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.073412 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.074551 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.083299 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.083491 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084452 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084621 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084668 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084841 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.085752 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.090727 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.090955 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.091229 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.091240 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.092566 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.094769 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.097555 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.097779 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.098800 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.105602 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.107064 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.113713 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.113866 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.113936 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.114117 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.114354 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.114574 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.114820 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.115034 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.115084 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.115304 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.115531 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.115814 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.116041 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.120445 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.121539 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.129660 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.129882 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130086 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130287 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130499 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130713 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130903 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133760 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133848 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.134095 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.134264 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.135215 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.136847 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.137044 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.137259 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.137449 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.137636 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.137852 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.138072 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.138296 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.141462 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.142797 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.147301 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.148128 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.148815 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.149039 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.149243 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.150207 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.150213 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.150417 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.150627 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.150825 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.153579 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.162552 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.162830 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.170485 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.174096 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.177681 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.178896 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.187251 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.187498 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.187525 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.187752 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.187752 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.187988 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188213 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188409 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188808 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188819 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.191847 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.192031 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.192209 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.194293 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195575 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195695 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196959 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.197189 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.198287 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.198599 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.201370 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.201983 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.202728 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.203312 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.207026 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.207139 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.207351 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.208454 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.222772 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.231575 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.231854 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232156 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232368 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232628 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.232851 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.234660 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.234880 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.235107 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.235307 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.235569 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.235791 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.236001 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.236200 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.236393 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.241880 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250006 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250262 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250530 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250754 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.250973 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.251278 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.251458 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.257869 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.258080 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260892 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261099 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261317 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261541 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261722 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264048 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264298 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264518 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264533 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264745 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.264797 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.265060 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.267530 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.272850 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.273113 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.274252 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277550 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277751 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277956 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.278956 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.284825 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.286068 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.291363 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.306712 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.306957 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.307165 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.307369 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.307577 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.307951 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.316804 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.317080 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.318185 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.327489 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.327788 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.328691 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.356429 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.356632 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.356873 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.357141 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.357342 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.357540 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.357802 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.358053 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.358342 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.361284 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.380322 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383265 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383455 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383715 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.383949 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.384181 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.384409 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.384665 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.387631 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.393561 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.396410 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.396632 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.397810 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.429975 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.430217 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.430414 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.430615 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.430862 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.431196 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.439373 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.439586 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.440614 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.449268 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.449463 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.450486 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.479598 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.479784 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.479990 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.480249 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.480482 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.480721 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.480921 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.481155 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.481355 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.481539 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.486771 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.500061 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.500336 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.503132 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.503317 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.503494 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.503683 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506508 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.509261 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.509464 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.512247 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.515049 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.516448 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.549115 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.549337 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.549587 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.549839 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.550052 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.550321 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.558422 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.558638 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.559602 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.568778 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.568987 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.575258 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.588295 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.588555 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.596468 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.596684 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.596942 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.597175 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.602470 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.602705 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.602895 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.605635 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.611001 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.611186 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.618909 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.621713 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.624378 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.624586 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.624752 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.624986 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.625206 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.625576 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.633338 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.633548 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.634680 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.050827 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.052450 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.056967 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.066730 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.068286 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.071591 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.072700 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.073110 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.077414 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.084990 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.086231 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.086465 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.086501 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.087785 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.088135 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.092250 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093030 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.100602 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.102116 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.103178 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.104351 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.104961 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.106124 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.106485 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.107704 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.110711 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.112231 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.113474 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.114847 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.117842 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.118119 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119366 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.122309 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.122577 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.123800 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124431 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124671 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.125941 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.128371 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.129217 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.130628 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.131145 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135665 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137650 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.140820 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143219 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143473 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.144794 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.149013 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152060 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152040 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152307 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152529 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.155411 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.155788 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.156744 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.156782 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.156999 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.157398 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.158056 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.167147 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.168589 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.168906 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.169131 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.169482 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.169744 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170020 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170262 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170359 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170910 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.179605 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.179889 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.179908 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.180977 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.185531 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.186612 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.189996 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.196224 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.197608 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.198058 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.198340 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.198621 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.199482 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.205073 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.209066 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.209367 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.209632 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.210403 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.210695 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.210784 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.218384 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.222147 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.222419 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.223410 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.224225 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.227854 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.958954 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.960554 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.969034 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.970532 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.982400 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.983040 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.992499 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.993928 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.995820 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.996168 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.997418 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.002020 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.005355 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.005589 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.006840 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.011767 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.012948 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.021377 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.022775 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.028287 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.786597 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.788067 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.799868 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.801249 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.802827 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.810201 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.813536 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.818866 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.820493 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.823523 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.830204 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.831758 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.832846 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.833109 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.834204 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.834546 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.841958 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.844409 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.845285 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.846009 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.846585 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.847500 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.850788 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.857594 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.857686 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.859200 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.859544 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.859755 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.861147 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.861280 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.866031 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.868636 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.869681 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.871097 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.876988 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.879637 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.881091 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.888110 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.888367 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.889299 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.889528 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.901051 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.901343 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.902590 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.909371 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.110266 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.112004 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.126595 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.135429 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.136888 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.148748 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.150147 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.958232 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.959697 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.965382 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.004433 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.014704 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.024849 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.034305 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.035933 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.048538 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.056936 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.066870 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.088972 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.102761 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.120074 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.122896 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.140978 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.142599 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.148833 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.155117 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.156533 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.167721 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.169018 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.176026 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.649024 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.650542 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.659906 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.668651 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.670262 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.688786 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.690209 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.700280 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.706808 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.707094 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.709374 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.718048 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.719621 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.728421 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.732072 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.525398 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.546368 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.555518 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.556984 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.577234 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.659633 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.659940 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.661229 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.665992 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.478948 1290 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.434494 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.436422 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.449321 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.458076 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:06.972422 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:07.046823 1424 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.302661 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.311481 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.364738 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:17.439291 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:48.697081 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:48.714049 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:54.174943 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:54.210510 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:57.182565 1378 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:08:10.482251 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:08:10.483875 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:08:10.490252 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:10:22.130061 1312 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:10:23.200037 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_2.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_2.log new file mode 100644 index 0000000..6a7ebbd --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_2.log @@ -0,0 +1,1629 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:46:18] KV cache is forced as chunk cache for decode server +[2026-08-31 09:46:18] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:46:18] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:46:21] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:46:21] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.86, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.5:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 2, 'tp_size': 32, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 680137396, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.21, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': None, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'decode', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': 16, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:45 TP18 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:45 TP18 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:50 TP18 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:50 TP18 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:46:50 TP20 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP20 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:50 TP19 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP19 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:50.442051 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:50.442131 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:50.442142 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:50.442183 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:15326 +I0831 09:46:50.442306 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:50.447149 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:50.451531 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:50.456059 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:50.461042 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:50.462805 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:50.462878 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:50.462916 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:50.468442 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:50.469667 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:50.469689 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:50.475536 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:50.476727 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:50.476747 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:50.482563 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:50.483769 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:50.483788 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:50.490281 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:50.491550 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:50.491652 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:50 TP18 EP2] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:50 TP16 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP16 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:51 TP23 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:51 TP23 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:52 TP17 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:52 TP17 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:52 TP22 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:52 TP22 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:53 TP21 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:53 TP21 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:53 TP20 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:53 TP20 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:54.060777 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:54.060835 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:54.060844 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:54.060871 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:16367 +I0831 09:46:54.060938 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:54.085126 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:54.091691 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:54.099220 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:54.107723 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:54.111367 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:54.111461 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:54.111516 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.119665 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:54.121074 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:54.121112 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.147588 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:54.148952 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:54.148981 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.175715 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:54.177098 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:54.177124 231 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:46:54 TP16 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:54 TP16 EP2] Multimodal data loading enabled with 16 worker threads (auto). +I0831 09:46:54.203663 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:54.205026 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:54.205156 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:54 TP20 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:54.460511 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:54.460589 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:54.460601 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:54.460634 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:15601 +I0831 09:46:54.460719 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:54.465137 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:54.469314 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:54.473737 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:54.478446 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:54.480096 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:54.480158 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:54.480195 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.485212 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:54.486311 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:54.486331 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.511622 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:54.512979 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:54.513012 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.539619 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:54.541008 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:54.541033 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.567591 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:54.568961 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:54.569113 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:54 TP16 EP2] Init torch distributed begin. +[2026-08-31 09:46:54 TP19 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:54 TP19 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:55.033336 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:55.033444 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:55.033456 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:55.033485 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:16537 +I0831 09:46:55.033583 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:55.057971 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:55.064479 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:55.071283 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:55.078248 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:55.080895 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:55.080996 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:55.081049 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.088224 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:55.089632 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:55.089661 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.115549 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:55.116875 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:55.116900 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.143584 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:55.144919 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:55.144943 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.171591 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:55.172920 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:55.173059 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:55 TP19 EP2] Init torch distributed begin. +[2026-08-31 09:46:55 TP17 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:55 TP17 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:46:56 TP23 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:56 TP23 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:56.217628 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:56.217694 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:56.217705 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:56.217730 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:15727 +I0831 09:46:56.217824 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:56.243990 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:56.250231 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:56.257269 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:56.264249 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:56.266896 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:56.266986 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:56.267048 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.274260 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:56.275656 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:56.275686 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.299576 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:56.300894 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:56.300925 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.327828 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:56.329331 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:56.329370 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.355741 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:56.357115 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:56.357239 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:56 TP17 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:56.395787 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:56.395848 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:56.395857 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:56.395882 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:15885 +I0831 09:46:56.395977 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:56.417263 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:56.422833 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:56.428838 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:56.435765 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:56.438443 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:56.438535 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:56.438591 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.445837 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:56.447266 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:56.447299 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.471720 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:56.473146 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:56.473174 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.499622 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:56.500993 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:56.501019 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.508332 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:56.509668 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:56.509795 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:56 TP23 EP2] Init torch distributed begin. +[2026-08-31 09:46:57 TP22 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:57 TP22 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:57.255920 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:57.255980 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:57.255988 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:57.256012 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:15467 +I0831 09:46:57.256090 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:57.277271 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:57.282449 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:57.287949 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:57.294042 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:57.296411 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:57.296499 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:57.296547 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.303107 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:57.304409 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:57.304435 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.331566 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:57.332916 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:57.332964 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.359781 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:57.361215 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:57.361243 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.387676 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:57.389057 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:57.389196 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:57 TP22 EP2] Init torch distributed begin. +[2026-08-31 09:46:58 TP21 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:58 TP21 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:58.326085 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:58.326146 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:58.326154 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.7 port: 12001 +I0831 09:46:58.326179 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.7:16782 +I0831 09:46:58.326263 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:58.330185 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:58.333969 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:58.337812 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:58.341971 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:58.343374 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:58.343430 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:58.343458 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:58.347656 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:58.348662 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:07 +I0831 09:46:58.348678 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:58.375608 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:58.376935 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:07 +I0831 09:46:58.376966 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:58.403645 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:58.405005 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:07 +I0831 09:46:58.405030 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:58.431660 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:58.433024 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:07 +I0831 09:46:58.433159 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:58 TP21 EP2] Init torch distributed begin. +[2026-08-31 09:47:09 TP23 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP21 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP22 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP20 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP19 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP18 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP17 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP16 EP2] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP23 EP2] Init torch distributed ends. elapsed=13.33 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP21 EP2] Init torch distributed ends. elapsed=11.40 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP22 EP2] Init torch distributed ends. elapsed=12.45 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP20 EP2] Init torch distributed ends. elapsed=15.63 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP19 EP2] Init torch distributed ends. elapsed=14.66 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP18 EP2] Init torch distributed ends. elapsed=19.35 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP17 EP2] Init torch distributed ends. elapsed=13.48 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP16 EP2] Init torch distributed ends. elapsed=15.27 s, mem usage=0.49 GB +[2026-08-31 09:47:10 TP20 EP2] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP21 EP2] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP18 EP2] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP19 EP2] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP23 EP2] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP20 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP21 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP18 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP16 EP2] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP22 EP2] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP19 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP23 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP16 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP22 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP17 EP2] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP17 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP20 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP21 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP18 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP23 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP19 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP20 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP21 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP18 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP23 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP19 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP16 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP22 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP17 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP16 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP22 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP17 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:12 TP23 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP21 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP20 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP18 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP22 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP17 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP16 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP19 EP2] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:51:18 TP17 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP18 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP18 EP2] Load weight end. elapsed=248.05 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP17 EP2] Load weight end. elapsed=248.03 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP19 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP19 EP2] Load weight end. elapsed=248.10 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP23 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP23 EP2] Load weight end. elapsed=248.11 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP20 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP20 EP2] Load weight end. elapsed=248.11 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP16 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP16 EP2] Load weight end. elapsed=248.14 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP21 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP21 EP2] Load weight end. elapsed=248.17 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:18 TP22 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:18 TP22 EP2] Load weight end. elapsed=248.22 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP23 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP17 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP23 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP17 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP18 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP16 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP18 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP16 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP20 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP20 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP21 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP21 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP19 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP19 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP22 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP22 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:23 TP23 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP23 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP23 EP2] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP17 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP17 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP17 EP2] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP16 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP16 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP20 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP16 EP2] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP20 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP20 EP2] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP18 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP18 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP18 EP2] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP21 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP21 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP21 EP2] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP22 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP19 EP2] Init torch distributed begin. +[2026-08-31 09:51:23 TP19 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP22 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP19 EP2] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP22 EP2] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP17 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP23 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP20 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP16 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP21 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP18 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP19 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP22 EP2] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:24 TP23 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP22 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP21 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP20 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP19 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP18 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP17 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP16 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP23 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP20 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP21 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP22 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP18 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP19 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP17 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP16 EP2] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP17 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP23 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP21 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP19 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP16 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP20 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP22 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP18 EP2] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP17 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP17 EP2] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP16 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP16 EP2] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP20 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP21 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP23 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP18 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP23 EP2] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP20 EP2] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP21 EP2] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP18 EP2] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP22 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP19 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP22 EP2] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP19 EP2] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:27 TP23 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP23 EP2] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP17 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP17 EP2] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP16 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP16 EP2] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP22 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP22 EP2] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP18 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP18 EP2] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP21 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP21 EP2] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP20 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP20 EP2] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP19 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP19 EP2] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP23 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP16 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP17 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP22 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP20 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP21 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP19 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP18 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP16 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp16_pp0_dp0.json +[2026-08-31 09:51:27 TP23 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp23_pp0_dp0.json +[2026-08-31 09:51:27 TP17 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp17_pp0_dp0.json +[2026-08-31 09:51:27 TP19 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp19_pp0_dp0.json +[2026-08-31 09:51:27 TP20 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp20_pp0_dp0.json +[2026-08-31 09:51:27 TP21 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp21_pp0_dp0.json +[2026-08-31 09:51:27 TP18 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp18_pp0_dp0.json +[2026-08-31 09:51:27 TP22 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp22_pp0_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/8 [00:000, gen1: 1885->0, gen2: 1109551->0 +[2026-08-31 09:55:11 TP16 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109542->0 +[2026-08-31 09:55:11 TP19 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109556->0 +[2026-08-31 09:55:11 TP18 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109552->0 +[2026-08-31 09:55:11 TP20 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109554->0 +[2026-08-31 09:55:11 TP17 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109545->0 +[2026-08-31 09:55:11 TP23 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109553->0 +[2026-08-31 09:55:11 TP21 EP2] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1885->0, gen2: 1109550->0 +I0831 09:55:20.389782 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.390061 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.419389 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.428344 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.428574 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.428817 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.429031 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.440006 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.479174 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.479463 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.488235 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.488521 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.488790 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.489022 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.497041 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.497342 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.497550 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.503330 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.526938 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.533185 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.533437 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.533702 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.533886 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.534098 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.534343 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.534549 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.534762 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.534957 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.535198 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.538502 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.569362 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.581574 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.581797 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.582028 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.582281 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.582476 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.582733 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.582983 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.583158 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.583364 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.583575 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.586745 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.586962 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.587145 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.587324 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.598692 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.600116 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.607966 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.609287 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623265 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623531 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623734 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624528 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.633028 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.633610 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.633864 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.634748 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.639947 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.641001 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.641209 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.641448 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.641741 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.641922 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642115 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642343 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642593 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642825 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.643039 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.643322 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.643608 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.643873 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.644191 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.644433 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.644629 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.652689 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.652976 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.653179 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.655192 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.656639 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.664377 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.665855 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.679507 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.679710 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.679888 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.683935 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.687357 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691422 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691614 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691888 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.692160 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.692667 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.694958 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.695168 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.695453 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.695626 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.695847 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.696107 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.696322 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.696558 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.696764 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697016 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697317 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697510 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697703 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697964 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.706028 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.708133 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.709663 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.717566 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.718804 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.719025 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.719069 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.719303 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.719583 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.719811 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.720000 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.720235 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.720418 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.726331 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.726809 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.728314 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.732609 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.732792 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.732982 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.736752 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.739979 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.743127 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.744517 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.748708 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.749934 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.756959 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.759959 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.764446 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.765848 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.773859 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.775375 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.788447 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.788645 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.788851 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.790017 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.792508 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.795426 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.795586 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.795608 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.795848 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.796068 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.796469 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.796727 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.796938 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.797152 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.797315 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.800185 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.800431 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.801568 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.801851 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.802876 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.815640 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818439 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818667 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818899 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819087 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819056 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819264 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819458 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.820525 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.829185 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.830528 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.830555 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.844017 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.844266 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.844475 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.848017 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.849370 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.854679 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858503 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858780 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.858978 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.859185 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.860999 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.861912 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.862143 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.862362 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.862540 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.862756 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.862984 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863149 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863404 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863612 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.863817 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.864049 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.872650 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.877676 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.879071 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.886721 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.888245 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.901793 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.901997 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.902251 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.906242 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.909354 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.912515 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.915729 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.916857 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918671 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918951 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.919147 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.927239 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.927441 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.927675 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.927891 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928084 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928313 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928573 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928817 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.928989 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929204 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929391 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929658 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929875 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930081 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930259 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930493 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930723 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.930994 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.931262 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.931465 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.931699 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.932968 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.934365 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.934693 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938832 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.940141 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.942241 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.943632 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.947077 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.956646 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.956849 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.957031 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.960659 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.961066 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.961436 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.961644 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.962580 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.962707 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.962932 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.963178 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.963369 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.963624 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.963655 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.963891 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.964072 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.964265 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.964478 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.964689 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.964900 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.964923 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.965200 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.970238 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.975839 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.976016 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.976253 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.977358 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.993975 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.994225 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.994504 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.051234 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.051466 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.051659 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.051868 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.052086 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.052313 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.052525 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.052711 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.052963 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.053205 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.053447 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.053617 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.053820 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.054004 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.067412 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.068804 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.071743 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.071973 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.072129 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.072391 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.072588 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.072798 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.073246 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.074728 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.080760 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.093803 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.094029 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.094245 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.094991 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.096951 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.097195 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.097421 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.097637 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.097889 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.098109 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.098309 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.098498 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.098706 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.103470 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.103681 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.104800 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.122202 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.123628 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.128054 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.135717 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.146438 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.146651 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.150314 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.153502 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.154448 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.154708 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.154913 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.155144 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.155380 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.155583 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.155805 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.156023 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.156288 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.156510 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.156726 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.156940 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157166 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157385 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157619 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157670 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157878 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158146 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158377 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158594 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158838 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.159097 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.161520 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.161799 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.162775 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.180543 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.181859 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.186628 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.191637 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.193490 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.194458 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.194657 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.194840 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195010 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195191 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195398 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195670 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.195854 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196098 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196327 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196521 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.196703 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.199483 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.206674 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.206967 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.207239 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.210140 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.210356 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.210636 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.210841 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.210870 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.211659 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.213730 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.216931 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.218302 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.234748 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.236119 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.240729 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.242039 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.245394 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.249591 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260228 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260512 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261818 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.267735 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.268257 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.268445 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.268648 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.268849 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.269229 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.269250 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.269444 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.269658 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.269852 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.270043 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.270304 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.273021 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.275375 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.276731 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.285776 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.288422 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.288641 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.288856 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.289062 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.289255 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.289803 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.293408 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.297662 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.299058 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.305980 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.319343 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.319569 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.319775 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.320720 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321135 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321344 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321555 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321823 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.322009 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.322228 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.322430 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.322628 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.322912 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.323120 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.323385 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.326239 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.329548 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.329790 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.330904 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.366029 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.373543 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.373778 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.374001 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.374284 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.374519 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.374723 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.374940 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.375164 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.375378 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.375607 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.375818 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.376024 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.376221 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.376479 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.376688 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.376967 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.377188 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.377362 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.380093 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.390678 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.391947 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.397176 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.403314 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.404302 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.407048 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.407220 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.407401 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.407644 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.407853 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.408080 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.408270 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.408479 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.408680 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.408887 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.411631 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.419039 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.421741 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.421933 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.422142 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423189 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423266 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423497 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423739 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.423971 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.486858 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487138 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487383 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487589 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487821 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.488041 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.488301 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.488508 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.488736 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.488981 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.489239 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.489478 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.489687 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.492403 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.503239 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.505321 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.505811 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506064 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506297 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506515 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506639 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.507225 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.512076 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.518523 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.528060 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.528264 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.528460 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.528635 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.528842 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.529050 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.529269 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.529489 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.530781 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.530985 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.531194 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.532172 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.543068 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.543304 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.543525 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.544273 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.563292 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.564546 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.568601 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.571811 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572072 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572266 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572466 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572731 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573020 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573218 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573434 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573705 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573894 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574080 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574287 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574508 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574725 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574920 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.575125 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.575313 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.580428 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.588092 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.588302 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.588488 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.589375 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.599751 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.599978 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.600169 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.601043 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.601148 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.601394 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.604043 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.604285 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.604534 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.604764 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.604986 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.605189 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.605405 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.605639 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.608480 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616082 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616561 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.617889 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.618685 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.618960 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.619149 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.619158 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.620076 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.624464 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.626133 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.630627 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.641757 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.641935 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.642128 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.647500 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.650195 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.650360 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.650633 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.653045 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.654450 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.669288 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.669564 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.669816 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.670049 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.670270 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.670507 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.670729 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.670935 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.671169 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.692127 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.692445 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.692679 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.692909 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.693122 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.693363 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.693570 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.693778 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.709506 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.709733 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.710022 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.710256 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.710538 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.710850 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.751559 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.751823 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.752106 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.752322 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.752542 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.752826 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753067 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753317 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753602 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753845 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754106 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754323 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754530 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754739 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754981 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.755206 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.768657 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.772248 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.773437 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.777554 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.780529 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.780786 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.781141 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.781386 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.781617 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.781844 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.782091 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.782354 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.782594 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.782791 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789553 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789755 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789947 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.790967 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.795630 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.795910 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.796134 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.796188 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.796370 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.797215 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.817212 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.842069 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.847563 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.847852 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.848052 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.848320 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.848527 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.848753 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.848937 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.849124 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.857405 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.858656 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.859395 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.859663 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.859867 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.860785 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.863701 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.864998 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.871563 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.871784 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.872004 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.872831 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.877050 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.877291 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.877538 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.878408 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.884032 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.885281 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.890470 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.911402 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.911636 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.911919 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.912184 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.912398 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.912606 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.912817 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.913038 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.913283 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.913504 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.913697 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.913923 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.914135 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.914318 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.914541 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.939571 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.939831 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.940075 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.940284 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.940521 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.940722 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.940936 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.941195 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.941408 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.941866 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.955017 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.955257 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.955480 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.955695 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.956409 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.001794 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.002035 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.002280 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.002492 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.002683 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.002866 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.003084 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.003276 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.013396 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.013628 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.013832 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.014655 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.025296 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.025507 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.025704 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.026614 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.063818 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.064066 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.064276 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.064474 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.064698 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.064893 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.065145 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.065336 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.065604 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.065832 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.066041 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.066247 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.066468 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.066677 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.069391 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.089404 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.092020 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.092231 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.092432 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.092645 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.092887 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.093108 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.093327 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.093538 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.096168 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.106130 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.106340 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.106601 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.106793 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:22.107625 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.069204 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.069561 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.069854 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.070958 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.077625 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.077919 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.079227 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.087476 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.087776 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.092741 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093037 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093246 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093467 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.093726 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.094257 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.094635 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.099941 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.101393 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.105710 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.106354 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.107112 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.107739 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.109299 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.111032 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.114991 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.115228 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.116380 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.118458 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119727 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.120975 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.122388 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124136 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124214 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.124469 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.125629 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.127038 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.128728 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.130208 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.130492 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.131580 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.132933 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135360 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135735 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135982 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.136677 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137153 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137408 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.138784 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.141963 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143091 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143512 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.144642 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.147691 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.148185 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.149765 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.150655 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.150892 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152151 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.155004 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.156415 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.156525 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.159564 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.159806 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.161046 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.161477 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.161741 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.162329 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.162361 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.162899 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.163826 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.164906 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.166272 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.168093 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170166 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.171550 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.171701 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.171790 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.172983 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.173239 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.175730 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.183152 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.183378 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.183604 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.183918 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.184556 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.184767 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.185321 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.185911 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.190807 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.194675 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.196147 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.196573 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.196882 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.197978 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.202131 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.202418 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.203154 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.203464 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.207719 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.212302 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.213610 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.222185 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.223629 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.231596 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.233078 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.960529 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.962153 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.966674 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.974397 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.975914 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.980871 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.982986 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.984531 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.989137 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.991907 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.992146 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.992934 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.993211 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.993477 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.994282 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.997824 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.999943 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.001418 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.005551 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.006307 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.006561 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.007736 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.008074 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.015578 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.022670 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.790868 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.804502 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.805943 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.809028 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.814555 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.823580 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.830264 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.830554 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.830819 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.831782 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.832876 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.836138 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.837175 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.845525 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.848379 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.848606 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.849953 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.852414 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.854301 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.861433 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.862943 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.865459 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.867086 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.868844 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.869150 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.869469 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.875185 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.875445 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.876479 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.876682 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.877816 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.880976 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.888945 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.889173 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.890455 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.902449 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.903934 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.123337 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.124989 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.129344 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.136247 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.137770 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.005904 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.024395 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.027503 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.029084 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.035600 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.035835 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.037266 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.046516 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.050671 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.052263 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.063154 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.083256 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.083504 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.084694 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.072856 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.082291 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.090528 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.091086 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.099994 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.109236 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.120748 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.125721 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.131660 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.135792 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.141655 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.143086 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.148470 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.157781 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.172134 1444 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.181900 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.693564 1356 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.701994 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.703575 1314 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.714013 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.722348 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.743945 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.533216 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.534844 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.560500 1422 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.573289 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.515910 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.517546 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.418912 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.454602 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.200716 1288 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.307096 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.330665 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.354573 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.356086 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.385281 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.399897 1334 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:10.407789 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:24.013675 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:24.046932 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:24.095551 1380 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:10:22.542296 1400 rdma_endpoint.cpp:671] Received RDMA ready ACK. diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_3.log new file mode 100644 index 0000000..b87253b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/d_3.log @@ -0,0 +1,1617 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:46:18] KV cache is forced as chunk cache for decode server +[2026-08-31 09:46:18] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:46:18] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:46:21] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:46:21] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.86, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.5:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 3, 'tp_size': 32, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 505268607, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.21, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': None, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'decode', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': 16, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:46:21] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:47 TP27 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:47 TP27 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:50 TP26 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:50 TP26 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:51 TP30 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:51 TP30 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:52 TP28 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:52 TP28 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:52 TP31 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:52 TP31 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:53 TP27 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:53 TP27 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:46:53 TP24 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:53 TP24 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:53.275849 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:53.275938 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:53.275950 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:53.275987 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16941 +I0831 09:46:53.276093 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:53.301103 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:53.308037 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:53.315229 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:53.322360 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:53.325075 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:53.325179 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:53.325240 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.332587 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:53.334034 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:53.334064 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.361531 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:53.362913 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:53.362957 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.389477 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:53.390874 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:53.390898 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:53.417480 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:53.418844 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:53.418987 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:53 TP27 EP3] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:46:54 TP29 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:54 TP29 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:46:54 TP26 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:54 TP26 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:54.833894 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:54.833966 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:54.833977 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:54.834007 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:15049 +I0831 09:46:54.834102 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:54.860841 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:54.869832 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:54.878794 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:54.887795 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:54.891593 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:54.891690 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:54.891738 229 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:46:54 TP25 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:46:54 TP25 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0831 09:46:54.899859 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:54.901293 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:54.901321 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.929502 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:54.930867 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:54.930891 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.957389 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:54.958730 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:54.958756 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:54.989378 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:54.990705 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:54.990820 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:54 TP26 EP3] Init torch distributed begin. +[2026-08-31 09:46:55 TP30 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:55 TP30 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:55.322562 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:55.322624 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:55.322634 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:55.322661 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16350 +I0831 09:46:55.322742 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:55.326256 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:55.329646 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:55.332926 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:55.336593 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:55.337806 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:55.337863 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:55.337891 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.341593 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:55.342464 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:55.342480 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.373494 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:55.374862 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:55.374894 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.401713 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:55.403201 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:55.403236 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:55.429553 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:55.431037 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:55.431169 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:55 TP30 EP3] Init torch distributed begin. +[2026-08-31 09:46:56 TP31 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:56 TP31 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:56.481407 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:56.481467 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:56.481475 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:56.481501 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16398 +I0831 09:46:56.481580 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:56.502246 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:56.506807 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:56.511315 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:56.516777 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:56.518833 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:56.518911 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:56.518970 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.524665 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:56.525875 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:56.525898 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.549511 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:56.550863 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:56.550886 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.577479 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:56.578835 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:56.578860 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.605449 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:56.606807 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:56.606927 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:56 TP31 EP3] Init torch distributed begin. +[2026-08-31 09:46:56 TP28 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:56 TP28 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:56.854842 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:56.854911 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:56.854919 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:56.854951 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16762 +I0831 09:46:56.855026 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:56.879662 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:56.886166 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:56.892737 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:56.899724 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:56.902391 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:56.902483 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:56.902541 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.929674 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:56.931222 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:56.931257 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.938660 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:56.940009 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:56.940042 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:56.947364 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:56.948719 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:56.948743 231 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:46:56 TP24 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:56 TP24 EP3] Multimodal data loading enabled with 16 worker threads (auto). +I0831 09:46:56.973483 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:56.974841 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:56.974979 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:56 TP28 EP3] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:57.197716 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:57.197793 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:57.197805 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:57.197832 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16211 +I0831 09:46:57.197935 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:57.220320 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:57.226791 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:57.233737 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:57.240689 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:57.243386 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:57.243480 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:57.243533 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.250662 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:57.252072 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:57.252100 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.277519 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:57.278877 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:57.278900 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.305449 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:57.306785 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:57.306808 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:57.333449 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:57.334770 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:57.334883 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:57 TP24 EP3] Init torch distributed begin. +[2026-08-31 09:46:59 TP29 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:59 TP29 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:59.228591 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:59.228652 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:59.228662 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:59.228684 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16328 +I0831 09:46:59.228766 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:59.250977 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:59.256434 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:59.261950 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:46:59.268224 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:46:59.270619 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:46:59.270704 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:46:59.270750 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:59.277304 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:46:59.278617 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:46:59.278641 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:59.305433 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:46:59.306778 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:46:59.306828 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:59.333496 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:46:59.334878 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:46:59.334903 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:46:59.361441 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:46:59.362818 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:46:59.362957 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:46:59 TP29 EP3] Init torch distributed begin. +[2026-08-31 09:46:59 TP25 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:46:59 TP25 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:46:59.959434 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:46:59.959506 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:46:59.959518 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.8 port: 12001 +I0831 09:46:59.959547 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.8:16375 +I0831 09:46:59.959649 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:46:59.983367 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:46:59.989773 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:46:59.996259 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:47:00.003240 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:47:00.005918 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:47:00.006012 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:47:00.006069 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.013214 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:47:00.014628 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:08 +I0831 09:47:00.014657 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.041399 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:47:00.042766 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:08 +I0831 09:47:00.042793 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.069399 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:47:00.070762 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:08 +I0831 09:47:00.070789 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:47:00.097364 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:47:00.098717 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:08 +I0831 09:47:00.098843 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:47:00 TP25 EP3] Init torch distributed begin. +[2026-08-31 09:47:09 TP31 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP29 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP30 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP28 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP27 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP25 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP24 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP26 EP3] CustomAllreduce is disabled because this process group spans across nodes. +[2026-08-31 09:47:09 TP31 EP3] Init torch distributed ends. elapsed=13.27 s, mem usage=0.45 GB +[2026-08-31 09:47:09 TP30 EP3] Init torch distributed ends. elapsed=14.45 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP29 EP3] Init torch distributed ends. elapsed=10.51 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP28 EP3] Init torch distributed ends. elapsed=12.90 s, mem usage=0.45 GB +[2026-08-31 09:47:09 TP27 EP3] Init torch distributed ends. elapsed=16.46 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP26 EP3] Init torch distributed ends. elapsed=14.89 s, mem usage=0.48 GB +[2026-08-31 09:47:09 TP24 EP3] Init torch distributed ends. elapsed=12.54 s, mem usage=0.49 GB +[2026-08-31 09:47:09 TP25 EP3] Init torch distributed ends. elapsed=9.78 s, mem usage=0.48 GB +[2026-08-31 09:47:10 TP26 EP3] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP26 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP24 EP3] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP29 EP3] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP28 EP3] Load weight begin. avail mem=82.13 GB +[2026-08-31 09:47:10 TP24 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP27 EP3] Load weight begin. avail mem=82.10 GB +[2026-08-31 09:47:10 TP31 EP3] Load weight begin. avail mem=82.13 GB +[2026-08-31 09:47:10 TP25 EP3] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP28 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP29 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP30 EP3] Load weight begin. avail mem=82.11 GB +[2026-08-31 09:47:10 TP26 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP31 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP27 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP25 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP30 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:47:10 TP26 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP24 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP28 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP27 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP24 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP29 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP31 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP30 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP28 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP25 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:47:10 TP27 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP31 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP29 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP30 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:10 TP25 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:47:12 TP31 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP30 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP29 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP28 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP26 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP24 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP25 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:47:12 TP27 EP3] multimem all-gather disabled because the TP group spans across nodes. +[2026-08-31 09:51:19 TP24 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP24 EP3] Load weight end. elapsed=248.90 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP30 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP30 EP3] Load weight end. elapsed=248.90 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP29 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP29 EP3] Load weight end. elapsed=248.94 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP25 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP25 EP3] Load weight end. elapsed=248.95 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP27 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP27 EP3] Load weight end. elapsed=248.98 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.52 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP28 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP28 EP3] Load weight end. elapsed=249.00 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.56 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP26 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP26 EP3] Load weight end. elapsed=249.05 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.53 GB, mem usage=58.58 GB. +[2026-08-31 09:51:19 TP31 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:51:19 TP31 EP3] Load weight end. elapsed=249.14 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=23.56 GB, mem usage=58.58 GB. +[2026-08-31 09:51:22 TP25 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP25 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP24 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP24 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP26 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP26 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP29 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP29 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP28 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP28 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP27 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP27 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:22 TP30 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:22 TP30 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:23 TP31 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:51:23 TP31 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:51:23 TP31 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP31 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP31 EP3] Load weight begin. avail mem=23.56 GB +[2026-08-31 09:51:23 TP25 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP25 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP25 EP3] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP26 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP26 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP26 EP3] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP29 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP29 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP27 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP29 EP3] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP28 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP27 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP28 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP27 EP3] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP28 EP3] Load weight begin. avail mem=23.56 GB +[2026-08-31 09:51:23 TP30 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP24 EP3] Init torch distributed begin. +[2026-08-31 09:51:23 TP30 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP24 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:51:23 TP30 EP3] Load weight begin. avail mem=23.53 GB +[2026-08-31 09:51:23 TP24 EP3] Load weight begin. avail mem=23.52 GB +[2026-08-31 09:51:23 TP24 EP3] Load weight end. elapsed=0.45 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP28 EP3] Load weight end. elapsed=0.45 s, type=DFlashDraftModel, avail mem=22.48 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP31 EP3] Load weight end. elapsed=0.48 s, type=DFlashDraftModel, avail mem=22.48 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP27 EP3] Load weight end. elapsed=0.46 s, type=DFlashDraftModel, avail mem=22.44 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP26 EP3] Load weight end. elapsed=0.46 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP30 EP3] Load weight end. elapsed=0.46 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP25 EP3] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:23 TP29 EP3] Load weight end. elapsed=0.47 s, type=DFlashDraftModel, avail mem=22.46 GB, mem usage=1.07 GB. +[2026-08-31 09:51:24 TP31 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP30 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP29 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP28 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP27 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP26 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP25 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP24 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:51:24 TP31 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP29 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP28 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP24 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP30 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP26 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP27 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP25 EP3] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-08-31 09:51:24 TP29 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP26 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP28 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP27 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP25 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP31 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP30 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP24 EP3] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-08-31 09:51:24 TP24 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP24 EP3] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP30 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP30 EP3] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP25 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP25 EP3] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP29 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP29 EP3] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP26 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP26 EP3] Memory pool end. avail mem=9.18 GB +[2026-08-31 09:51:24 TP27 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP31 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP27 EP3] Memory pool end. avail mem=9.16 GB +[2026-08-31 09:51:24 TP31 EP3] Memory pool end. avail mem=9.20 GB +[2026-08-31 09:51:24 TP28 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-08-31 09:51:24 TP28 EP3] Memory pool end. avail mem=9.20 GB +[2026-08-31 09:51:27 TP24 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP24 EP3] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP29 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP29 EP3] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP25 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP25 EP3] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP31 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP31 EP3] Memory pool end. avail mem=8.36 GB +[2026-08-31 09:51:27 TP28 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP28 EP3] Memory pool end. avail mem=8.36 GB +[2026-08-31 09:51:27 TP26 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP27 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP26 EP3] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP27 EP3] Memory pool end. avail mem=8.32 GB +[2026-08-31 09:51:27 TP30 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-08-31 09:51:27 TP30 EP3] Memory pool end. avail mem=8.33 GB +[2026-08-31 09:51:27 TP29 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP24 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP25 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP31 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP28 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP26 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP27 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP30 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:51:27 TP25 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp25_pp0_dp0.json +[2026-08-31 09:51:27 TP24 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp24_pp0_dp0.json +[2026-08-31 09:51:27 TP29 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp29_pp0_dp0.json +[2026-08-31 09:51:27 TP26 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp26_pp0_dp0.json +[2026-08-31 09:51:27 TP27 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp27_pp0_dp0.json +[2026-08-31 09:51:27 TP28 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp28_pp0_dp0.json +[2026-08-31 09:51:27 TP30 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp30_pp0_dp0.json +[2026-08-31 09:51:27 TP31 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp31_pp0_dp0.json +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/8 [00:000, gen1: 1887->0, gen2: 1109547->0 +[2026-08-31 09:55:11 TP26 EP3] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109546->0 +[2026-08-31 09:55:11 TP29 EP3] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109551->0 +[2026-08-31 09:55:11 TP27 EP3] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109549->0 +[2026-08-31 09:55:11 TP30 EP3] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109548->0 +[2026-08-31 09:55:11 TP24 EP3] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109548->0 +[2026-08-31 09:55:11 TP25 EP3] Freezing GC in Scheduler process. gen0: 431->0, gen1: 1885->0, gen2: 1109548->0 +[2026-08-31 09:55:11 TP31 EP3] Freezing GC in Scheduler process. gen0: 428->0, gen1: 1887->0, gen2: 1109543->0 +I0831 09:55:20.416775 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.425752 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.426033 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.426337 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.464781 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.465111 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.465358 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.465586 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.474313 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.479825 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.488474 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.508623 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.508901 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.527741 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.527966 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.528239 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.528429 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.528673 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.528937 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.529152 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.532305 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.532547 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.541415 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.564239 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.570101 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.570287 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.570577 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.570765 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.571069 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.571265 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.571511 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.571764 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.571997 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.572227 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.572424 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.575379 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.578318 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.608464 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.611589 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622469 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.622741 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623000 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623221 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623469 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.623736 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624046 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624258 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624490 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624712 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.624909 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.636165 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.636402 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.637063 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.637297 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.637539 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.642449 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.643903 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.652546 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.653879 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.667927 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668188 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668247 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668433 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668524 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668732 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.668946 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.669646 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.676085 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.677587 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.689114 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.689381 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.689643 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.689877 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.690128 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.690338 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.690579 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.690802 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691035 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691287 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.691512 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.693847 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.695166 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697499 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697731 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.697947 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.698190 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.698362 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.698614 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.701491 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.703081 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.704466 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715263 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715521 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.715857 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.716099 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.716323 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.716780 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.716938 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.717162 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.717190 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.723069 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.724363 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.724641 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.724915 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.725136 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.725389 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.725617 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.725844 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.726099 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.726598 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.726842 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.728756 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.728982 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.729183 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731026 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.731784 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.732206 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.733026 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.737356 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.739933 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.754787 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.756064 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.756135 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.764190 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.765619 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.778803 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.779027 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.779224 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.780193 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.786094 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.787331 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.805100 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.806674 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.810607 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.815500 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816237 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816552 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816744 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816807 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.816992 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.817312 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.817551 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.817740 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.817986 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818254 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818451 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818639 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.818925 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819168 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819378 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819641 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.819875 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.827273 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.827497 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.831248 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.834102 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.837092 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.840888 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.842126 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.847182 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.851776 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.852077 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.852360 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.852594 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.852901 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.853142 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.853353 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.853626 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.853864 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.854027 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.854230 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.854400 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.864312 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.865685 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.873581 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.875046 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.882793 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.883075 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.883325 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.883569 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.883769 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.883991 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.884274 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.884517 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.884793 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.885041 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.885269 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.887804 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.888037 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.888228 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.891654 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.897347 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.900292 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.900466 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.901690 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908543 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908771 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.908975 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.912832 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.914143 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.918372 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.919631 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.919705 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.920871 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.927799 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.929131 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.934733 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935034 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935294 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935518 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935736 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.935974 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936205 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936223 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936426 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936442 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936632 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936697 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936913 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.936925 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937126 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937346 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937556 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937584 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.937887 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938149 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938400 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938656 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.938926 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.939168 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.939836 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.940047 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.944094 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.947105 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.952620 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.952816 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953696 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953886 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.953925 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.954169 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.954373 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.954623 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.957643 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.958901 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.965082 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.967780 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968022 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.968190 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.969254 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.970867 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.971161 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.971442 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.971832 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.972090 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.972370 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.972599 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.972883 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.973071 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.973294 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.973551 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.973788 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.974785 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.976137 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.980207 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.981482 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.983824 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.985119 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.996937 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.997248 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.997524 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.998191 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.998545 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.998783 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:20.998994 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.004994 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.005652 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.006233 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.008755 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.008972 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.012228 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.012902 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.013782 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.014192 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.033735 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.033958 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.034199 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.034437 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.034668 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.034927 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.037979 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.042171 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.042433 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.042629 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.042867 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.043149 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.043354 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.043555 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.043743 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.044005 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.046831 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.048101 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.050716 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.051002 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.051213 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.053853 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.056447 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.056656 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.057792 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.059825 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.060029 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.060200 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.060412 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.060629 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.061100 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.061764 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.081771 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.082039 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.082231 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.083001 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.083956 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084221 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084468 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084695 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.084916 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.085119 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.085353 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.085546 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.088536 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.089871 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.095471 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.096627 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.117854 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.118131 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.118330 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.118548 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.118777 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.119012 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.119282 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.119505 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.130977 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.131263 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.131523 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.131747 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.131946 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132160 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132407 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132563 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132683 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132820 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.132907 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133008 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133164 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133248 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133420 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133708 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133885 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.133925 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.134169 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.134387 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.134608 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.134888 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.135116 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.138836 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157488 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157722 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.157948 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.158921 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.164453 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.164729 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.165014 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.165274 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.165467 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.165699 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.165788 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.165961 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.166224 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.166489 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.166738 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.167064 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.167301 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.167539 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.171341 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.172681 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.173473 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.174754 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188519 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188732 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.188923 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189128 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.189332 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.192152 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.203169 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.205626 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.205847 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.206046 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.208559 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.211330 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.214044 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.214282 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.215436 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.218612 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.218860 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.223786 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.224107 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.224326 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.224488 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.224655 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.227300 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.227509 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.227769 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.237212 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.239604 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.239861 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.239871 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.240037 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.240097 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.240229 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.240450 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.240942 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.242976 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.246595 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.248018 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.254022 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.255276 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.257839 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260409 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260586 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260769 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.260977 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261178 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.261406 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.273742 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277055 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277271 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277467 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277681 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.277882 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.278074 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.278328 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.278523 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.294344 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.294579 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.294842 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295167 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295358 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.295665 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.298496 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.298704 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.298897 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.299086 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.299296 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.299803 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.309517 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.309751 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.309988 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.310194 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.310442 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.310645 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.310974 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.311182 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.311398 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.314028 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.316602 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.316857 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.317124 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.317389 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321010 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321230 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.321445 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.322372 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.328059 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.329406 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.335130 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.336476 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.337325 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.337505 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.342628 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.342839 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.343031 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.343222 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.343432 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.343657 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.343865 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.344055 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.344285 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.344522 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.350725 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.352120 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.352126 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.352329 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.352630 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.352830 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.353056 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.355733 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.365850 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.368463 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.368690 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.368885 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.371466 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.371742 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.374383 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.376972 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.378342 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.385888 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.386094 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.395813 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.396057 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.396238 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.399091 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.399296 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.399501 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.399684 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.402390 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.405061 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.412547 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.412734 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.415347 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.415594 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.415782 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.425249 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.432561 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.432735 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.432932 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.433105 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.442763 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.442966 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.445631 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.450618 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.450891 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.451103 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.451267 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.470613 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.480419 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.483050 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.485652 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.485914 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.486116 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.486361 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.486616 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.486850 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487064 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487257 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487529 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487763 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.487983 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.488245 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.490931 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.491127 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.506035 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.507261 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.511257 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.513300 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.513547 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.513748 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.513957 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.514217 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.514489 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.514698 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.514900 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.515183 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.515379 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.520321 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.521869 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.528757 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572172 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572400 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572594 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572815 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.572997 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573315 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573519 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.573771 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574000 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574285 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.574565 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.577176 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.594055 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.596626 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.596813 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.597023 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.597244 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.597442 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.597613 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.597836 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.598064 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.611423 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.612629 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.615610 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.615788 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.615975 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616178 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616392 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.616590 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.617746 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.619493 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.623521 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.634812 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.635015 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.635222 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.636057 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.641997 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.646198 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.646417 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.646616 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.647497 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652038 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652275 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652482 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652693 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.652873 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.653093 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.653368 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.653601 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.653816 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.656466 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.656677 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.656893 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.657189 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.662184 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.663385 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.668453 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.669533 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.674458 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.674639 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.674851 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.675069 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.675266 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.675523 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.675722 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.678669 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.681473 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.682921 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.685665 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.685868 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.686054 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.686915 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.696573 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.696784 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.696997 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.697842 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.719820 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.720062 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.720247 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.720505 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.720701 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.720894 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.721095 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.721297 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.733603 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.733827 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.734054 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.734273 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.735129 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.752892 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753108 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753314 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753489 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753736 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.753937 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.754252 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.787695 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.787920 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.788157 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.788362 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.788570 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.788786 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789012 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789217 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789418 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789634 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.789842 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.790067 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.790290 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.792824 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.809734 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.809953 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.810249 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.810469 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.810719 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.810993 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.811251 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.811453 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.816598 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.817996 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.857949 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.858157 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.858404 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.858596 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.858798 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.859022 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.859228 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.861855 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.871672 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.871891 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.872104 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.872299 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.873139 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.890875 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.891093 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.891294 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.891475 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.891686 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.891987 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.894654 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.916100 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.925916 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.926190 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.926398 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.926656 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.926878 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.927260 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.927554 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.927762 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.928028 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.928303 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.928565 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.928785 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.931718 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.946440 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.949018 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.949263 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.949535 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.949769 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.949988 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.950168 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.950556 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.955679 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 09:55:21.957017 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.064396 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.066016 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.077858 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.079484 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.081700 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.083266 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.085476 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.094179 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.095248 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.096788 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.097198 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.098764 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.100242 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.106786 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.108006 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.108242 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.108726 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.112879 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.114436 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.116112 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119107 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.119356 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.120705 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.126698 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.128352 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.129141 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.129431 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.130620 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.134586 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135649 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135739 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.135910 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137075 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.137184 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.139484 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.140954 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.143078 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.149737 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.150113 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.150663 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.151053 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.151162 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152305 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.152657 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.153827 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.158000 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.160044 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.160281 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.161724 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.163936 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.164181 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.165442 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.167531 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.168232 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.169013 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.170014 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.171850 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.173292 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.173564 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.173650 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.175016 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.176298 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.182139 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.183545 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.184733 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.184959 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.186297 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.189445 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.189683 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.191000 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.193118 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.193424 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.194868 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.195500 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.199213 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.207247 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.207465 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.208675 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.210290 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.210516 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.211809 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.216048 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.219287 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.220803 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.227166 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.227583 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.227818 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.229043 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.235805 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.237216 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.243009 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.250964 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:32.252275 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.968391 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.968637 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.970105 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.976513 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.980948 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.991892 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.993396 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.994191 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.995620 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:33.997649 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.004747 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.006112 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.006131 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.006424 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.007838 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.014101 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.031095 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.032590 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.038421 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:34.045575 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.801084 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.806982 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.818395 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.819331 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.819904 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.820710 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.828332 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.830559 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.842904 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.843204 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.844201 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.844408 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.854384 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.855816 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.857098 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.858050 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.858577 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.859305 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.859563 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.860759 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.864995 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.867637 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.868995 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.871797 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.873332 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.873799 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.875416 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.877663 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.884152 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.887104 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.888538 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.895428 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.896799 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.897631 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.899080 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.905704 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.909914 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:35.911327 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:38.141381 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.970695 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.972195 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:39.978142 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.014868 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.016410 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.026808 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.028273 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.048272 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.063985 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.064741 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.065464 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.066243 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:40.070612 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.078004 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.087821 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.096825 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.100546 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.107633 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.113929 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.121456 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.131799 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.133131 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.141551 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.143050 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.147181 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.162127 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.166158 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.167639 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.173517 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.174873 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.188687 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:42.189926 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.652436 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.671056 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.721783 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.737098 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.744935 1421 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:45.754668 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.514088 1357 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.529450 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.536650 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.539911 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.552011 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.553426 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.563019 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.568687 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.577418 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.586836 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:48.588416 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.571321 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.571593 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.572917 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.577956 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:52.631237 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.509544 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.526633 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.554049 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:06:56.571264 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:00.449039 1333 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.128557 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.149467 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.166285 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.181497 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.190620 1399 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:04.195330 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:07.009174 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:07.046696 1443 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:07.057155 1311 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:07.061699 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:17.448276 1289 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:20.623781 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0831 10:07:39.366431 1379 rdma_endpoint.cpp:671] Received RDMA ready ACK. diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_0.log new file mode 100644 index 0000000..2a1e054 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_0.log @@ -0,0 +1,1126 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:41:39] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:41:39] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:41:39] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 09:41:42] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:41:42] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 0, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 638571199, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:41:47] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:41:47] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:41:48] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:41:48] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:41:48] CommonKVBootstrapServer started successfully on 0.0.0.0:28800 +[2026-08-31 09:41:48] No HuggingFace chat template found +[2026-08-31 09:41:48] No chat template found, defaulting to 'string' content format +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:10 PP0 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:10 PP0 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:11 PP0 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:11 PP0 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:11 PP0 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:11 PP0 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:11 PP1 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:11 PP1 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:11] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:11] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:11 PP1 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:11 PP1 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +/opt/kimi-dflash/python/sglang/srt/utils/hf_transformers/tokenizer.py:433: UserWarning: Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead. + warnings.warn( +[2026-08-31 09:42:11] Applying special tokens cache patch for Kimi tokenizer: +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:12 PP1 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:12 PP1 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:13 PP0 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:13 PP0 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:13 PP1 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:13 PP1 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:13 PP0 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:13 PP0 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:14.001024 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:14.001106 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:14.001118 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:14.001147 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15633 +I0831 09:42:14.001235 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:14.024324 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:14.030330 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:14.036304 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:14.042698 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:14.045166 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:14.045266 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:14.045321 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.052469 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:14.054062 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:14.054092 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.078192 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:14.079604 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:14.079630 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.106220 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:14.107734 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:14.107759 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.134217 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:14.135622 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:14.135756 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:14 PP0 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:42:14 PP0 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:14 PP0 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:15 PP1 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:15 PP1 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:15.252722 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:15.252828 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:15.252843 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:15.252884 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15404 +I0831 09:42:15.253016 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:15.278276 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:15.287340 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:15.296291 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:15.305267 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:15.308928 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:15.309022 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:15.309080 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.317206 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:15.318635 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:15.318701 231 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:15.330526 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:15.330598 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:15.330610 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:15.330632 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15481 +I0831 09:42:15.330703 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:15.346210 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:15.347621 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:15.347664 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.355603 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:15.361902 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:15.368883 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:15.374140 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:15.375512 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:15.375535 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.393994 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:15.396752 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:15.396847 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:15.396904 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.404119 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:15.405654 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:15.405685 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.406227 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:15.407616 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:15.407771 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:15 PP0 TP3 EP3] Init torch distributed begin. +I0831 09:42:15.434293 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:15.435683 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:15.435711 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.462285 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:15.463676 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:15.463701 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.490237 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:15.491627 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:15.491758 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:15 PP1 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:42:15 PP0 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:15 PP0 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:15.821251 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:15.821326 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:15.821336 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:15.821364 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16114 +I0831 09:42:15.821456 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:15.825884 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:15.829946 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:15.833931 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:15.838352 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:15.839915 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:15.839978 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:15.840011 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.844470 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:15.845568 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:15.845587 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.869985 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:15.871356 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:15.871384 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.898451 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:15.899931 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:15.899964 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:15.926182 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:15.927582 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:15.927724 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:15 PP0 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:42:16 PP1 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:16 PP1 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:16.220542 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:16.220610 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:16.220620 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:16.220643 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16939 +I0831 09:42:16.220736 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:16.224701 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:16.228497 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:16.232280 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:16.236037 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:16.237270 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:16.237324 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:16.237350 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.242811 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:16.243970 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:16.243991 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.270152 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:16.271543 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:16.271574 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.298205 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:16.299612 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:16.299636 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.326160 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:16.327560 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:16.327746 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:16 PP1 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:42:17 PP1 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:17 PP1 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:17.327736 235 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:17.327809 235 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:17.327821 235 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:17.327844 235 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16171 +I0831 09:42:17.327932 235 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:17.352758 235 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:17.359525 235 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:17.385878 235 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:17.392753 235 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:17.395429 235 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:17.395522 235 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:17.395577 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.404488 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:17.406216 235 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:17.406298 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.430116 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:17.431460 235 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:17.431488 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.458174 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:17.459573 235 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:17.459597 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.486186 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:17.487546 235 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:17.487694 235 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:17 PP1 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:17 PP0 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:17 PP0 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:18 PP1 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:18 PP1 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:18.071556 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:18.071632 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:18.071643 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:18.071676 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:15356 +I0831 09:42:18.071780 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:18.096760 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:18.103821 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:18.110816 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:18.117731 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:18.120421 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:18.120519 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:18.120568 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.127691 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:18.129112 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:18.129140 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.154114 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:18.155493 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:18.155524 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.182120 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:18.183485 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:18.183514 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.210201 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:18.211604 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:18.211740 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:18 PP0 TP1 EP1] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:18.235406 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:18.235468 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:18.235477 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.1 port: 12001 +I0831 09:42:18.235499 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.1:16139 +I0831 09:42:18.235584 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:18.260762 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:18.267850 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:18.274878 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:18.281880 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:18.284564 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:18.284662 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:18.284718 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.310333 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:18.311815 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:01 +I0831 09:42:18.311846 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.319257 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:18.320643 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:01 +I0831 09:42:18.320672 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.346280 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:18.347678 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:01 +I0831 09:42:18.347702 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.374257 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:18.375630 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:01 +I0831 09:42:18.375762 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:18 PP1 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:42:24 PP0 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:24 PP1 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP0 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP1 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP0 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP0 TP2 EP2] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP0 TP3 EP3] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP0 TP1 EP1] sglang is using nccl==2.30.7 +[2026-08-31 09:42:26 PP0 TP0 EP0] Init torch distributed ends. elapsed=12.25 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP1 TP3 EP3] Init torch distributed ends. elapsed=8.90 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP1 TP2 EP2] Init torch distributed ends. elapsed=10.06 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP1 TP1 EP1] Init torch distributed ends. elapsed=8.01 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP1 TP0 EP0] Init torch distributed ends. elapsed=10.89 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP0 TP2 EP2] Init torch distributed ends. elapsed=10.46 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP0 TP3 EP3] Init torch distributed ends. elapsed=10.98 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP0 TP1 EP1] Init torch distributed ends. elapsed=8.17 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP0 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:26 PP0 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:26 PP0 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:26 PP0 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP1 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP1 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP1 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP0 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP0 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP1 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP1 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP1 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP0 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP0 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP1 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP1 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP0 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP0 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP1 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP1 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP1 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP1 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP0 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP1 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP1 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP0 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP1 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP0 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP0 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP1 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP0 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP0 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). + Multi-thread loading shards: 0% Completed | 0/96 [00:00, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:40 PP0 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:40 PP0 TP3 EP3] DFLASH PP rank 0: capture layers=[], projection columns=[], incoming=False +[2026-08-31 09:43:40 PP1 TP2 EP2] DFLASH PP rank 1: capture layers=[19], projection columns=[0], incoming=False +[2026-08-31 09:43:41 PP0 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP1 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP1 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP1 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP1 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP0 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP0 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP0 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP0 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP1 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP0 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP1 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP0 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP0 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP1 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP1 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP0 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 09:43:41 PP0 TP0 EP0] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP0 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP0 TP0 EP0] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP0 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 09:43:41 PP0 TP3 EP3] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP1 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP1 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP0 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 09:43:41 PP0 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 3.89 GB +[2026-08-31 09:43:41 PP1 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP1 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP0 TP1 EP1] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP0 TP2 EP2] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP1 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP1 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP1 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP1 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP0 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP0 TP3 EP3] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP1 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP1 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP1 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP1 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP0 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP0 TP2 EP2] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP0 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP1 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP0 TP1 EP1] Memory pool end. avail mem=33.60 GB +[2026-08-31 09:43:41 PP1 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP1 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP1 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP0 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP0 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP1 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP1 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP0 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP0 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp0_dp0.json +[2026-08-31 09:43:41 PP0 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp0_dp0.json +[2026-08-31 09:43:41 PP0 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp0_dp0.json +[2026-08-31 09:43:41 PP0 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp0_dp0.json +[2026-08-31 09:43:41 PP1 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP1 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp1_dp0.json +[2026-08-31 09:43:41 PP1 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp1_dp0.json +[2026-08-31 09:43:41 PP1 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp1_dp0.json +[2026-08-31 09:43:41 PP1 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp1_dp0.json +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:000, gen1: 310->0, gen2: 1075085->0 +[2026-08-31 09:45:28 PP0 TP3 EP3] Freezing GC in Scheduler process. gen0: 429->0, gen1: 310->0, gen2: 1075081->0 +[2026-08-31 09:45:28] Freezing GC in Tokenizer Manager process. gen0: 808->0, gen1: 4480->0, gen2: 1050512->0 +[2026-08-31 09:45:28] INFO: 127.0.0.1:46480 - "POST /freeze_gc HTTP/1.1" 200 OK +[2026-08-31 09:45:28] The server is fired up and ready to roll! +[2026-08-31 09:45:28 PP0 TP0 EP0] Freezing GC in Scheduler process. gen0: 451->0, gen1: 310->0, gen2: 1075272->0 +[2026-08-31 09:45:28 PP0 TP2 EP2] Freezing GC in Scheduler process. gen0: 430->0, gen1: 310->0, gen2: 1075082->0 +[2026-08-31 09:45:28] Freezing GC in Detokenizer Manager process. gen0: 780->0, gen1: 1427->0, gen2: 870548->0 +[2026-08-31 09:45:28 PP1 TP0 EP0] Freezing GC in Scheduler process. gen0: 574->0, gen1: 3157->0, gen2: 1072444->0 +[2026-08-31 09:45:28 PP1 TP1 EP1] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2949->0, gen2: 1072077->0 +[2026-08-31 09:45:28 PP1 TP3 EP3] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2949->0, gen2: 1072072->0 +[2026-08-31 09:45:28 PP1 TP2 EP2] Freezing GC in Scheduler process. gen0: 552->0, gen1: 2949->0, gen2: 1072077->0 +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[2026-08-31 09:45:33 PP0 TP1 EP1] Triton kernel '_fwd_kernel' took 1.28 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:33 PP0 TP3 EP3] Triton kernel '_fwd_kernel' took 1.29 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:33 PP0 TP0 EP0] Triton kernel '_fwd_kernel' took 1.35 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:33 PP0 TP2 EP2] Triton kernel '_fwd_kernel' took 1.39 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:45 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.65 +[2026-08-31 09:45:45 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:48 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.84 +[2026-08-31 09:45:48 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.85 +[2026-08-31 09:45:48] INFO: 174.1.60.1:54316 - "GET /health HTTP/1.1" 200 OK +[2026-08-31 09:46:23 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:46:23 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:46:24] INFO: 174.1.60.1:55396 - "GET /health HTTP/1.1" 200 OK +[2026-08-31 09:55:20 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP0 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP0 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP1 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP0 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP0 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP1 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP1 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP1 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:22] INFO: 174.1.60.1:53302 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 09:56:01] INFO: 174.1.60.1:38878 - "GET /model_info HTTP/1.1" 200 OK +[2026-08-31 10:05:07 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:05:07 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:05:08] INFO: 174.1.60.1:46878 - "GET /health HTTP/1.1" 200 OK +[2026-08-31 10:06:30] INFO: 174.1.60.1:42126 - "GET /model_info HTTP/1.1" 200 OK +[2026-08-31 10:06:32 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:32 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:32] INFO: 174.1.60.1:42138 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:33 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 369.65 +[2026-08-31 10:06:33 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 370.31 +[2026-08-31 10:06:34] INFO: 174.1.60.1:37432 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:35 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 421.95 +[2026-08-31 10:06:35 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 422.06 +[2026-08-31 10:06:36] INFO: 174.1.60.1:37438 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:38 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 301.34 +[2026-08-31 10:06:38 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 301.03 +[2026-08-31 10:06:38] INFO: 174.1.60.1:37454 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:39 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 451.24 +[2026-08-31 10:06:39 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 451.58 +[2026-08-31 10:06:40] INFO: 174.1.60.1:37464 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:42 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 365.69 +[2026-08-31 10:06:42 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 365.83 +[2026-08-31 10:06:42] INFO: 174.1.60.1:37480 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:45 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 215.09 +[2026-08-31 10:06:45 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.98 +[2026-08-31 10:06:45] INFO: 174.1.60.1:33548 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:48 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.37 +[2026-08-31 10:06:48 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.27 +[2026-08-31 10:06:48] INFO: 174.1.60.1:33562 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:52 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.30 +[2026-08-31 10:06:52 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.34 +[2026-08-31 10:06:52] INFO: 174.1.60.1:46888 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:06:56 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.33 +[2026-08-31 10:06:56 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.29 +[2026-08-31 10:06:56] INFO: 174.1.60.1:46900 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:00 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.86 +[2026-08-31 10:07:00 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.98 +[2026-08-31 10:07:00] INFO: 174.1.60.1:46912 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:04 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 208.11 +[2026-08-31 10:07:04 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 208.01 +[2026-08-31 10:07:04] INFO: 174.1.60.1:44436 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:06 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.45 +[2026-08-31 10:07:06 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.43 +[2026-08-31 10:07:07] INFO: 174.1.60.1:44438 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:10 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 230.19 +[2026-08-31 10:07:10 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 230.28 +[2026-08-31 10:07:10] INFO: 174.1.60.1:44454 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:14 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.29 +[2026-08-31 10:07:14 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.34 +[2026-08-31 10:07:14] INFO: 174.1.60.1:37142 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:17 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.33 +[2026-08-31 10:07:17 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.41 +[2026-08-31 10:07:17] INFO: 174.1.60.1:37154 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:20 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.80 +[2026-08-31 10:07:20 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.54 +[2026-08-31 10:07:20] INFO: 174.1.60.1:37164 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:23 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 223.70 +[2026-08-31 10:07:23 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 223.78 +[2026-08-31 10:07:24] INFO: 174.1.60.1:53468 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:27 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.49 +[2026-08-31 10:07:27 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.58 +[2026-08-31 10:07:28] INFO: 174.1.60.1:53474 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:30 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.42 +[2026-08-31 10:07:30 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.29 +[2026-08-31 10:07:31] INFO: 174.1.60.1:53490 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:35 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.93 +[2026-08-31 10:07:35 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.87 +[2026-08-31 10:07:35] INFO: 174.1.60.1:51800 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:39 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 210.91 +[2026-08-31 10:07:39 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 210.98 +[2026-08-31 10:07:39] INFO: 174.1.60.1:51810 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:41 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 343.78 +[2026-08-31 10:07:41 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.26 +[2026-08-31 10:07:41] INFO: 174.1.60.1:51824 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:43 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.42 +[2026-08-31 10:07:43 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.27 +[2026-08-31 10:07:44] INFO: 174.1.60.1:49412 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:46 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.68 +[2026-08-31 10:07:46 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.61 +[2026-08-31 10:07:46] INFO: 174.1.60.1:49420 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:48 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 294.18 +[2026-08-31 10:07:48 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 294.10 +[2026-08-31 10:07:48] INFO: 174.1.60.1:49430 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:51 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 230.00 +[2026-08-31 10:07:51 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 230.08 +[2026-08-31 10:07:52] INFO: 174.1.60.1:49434 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:54 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 359.61 +[2026-08-31 10:07:54 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 359.40 +[2026-08-31 10:07:54] INFO: 174.1.60.1:44544 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:57 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.96 +[2026-08-31 10:07:57 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 261.02 +[2026-08-31 10:07:57] INFO: 174.1.60.1:44558 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:07:59 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 317.79 +[2026-08-31 10:07:59 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 317.79 +[2026-08-31 10:07:59] INFO: 174.1.60.1:44566 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:01 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.51 +[2026-08-31 10:08:01 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.35 +[2026-08-31 10:08:02] INFO: 174.1.60.1:44574 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:05 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.43 +[2026-08-31 10:08:05 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.50 +[2026-08-31 10:08:05] INFO: 174.1.60.1:56062 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:08 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 245.01 +[2026-08-31 10:08:08 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 245.01 +[2026-08-31 10:08:08] INFO: 174.1.60.1:56070 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:10 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 335.96 +[2026-08-31 10:08:10 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 335.92 +[2026-08-31 10:08:10] INFO: 174.1.60.1:56080 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:13 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.72 +[2026-08-31 10:08:13 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.76 +[2026-08-31 10:08:13] INFO: 174.1.60.1:46174 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:15 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 290.19 +[2026-08-31 10:08:15 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 290.30 +[2026-08-31 10:08:16] INFO: 174.1.60.1:46178 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:18 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 330.01 +[2026-08-31 10:08:18 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.83 +[2026-08-31 10:08:18] INFO: 174.1.60.1:46186 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:21 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.44 +[2026-08-31 10:08:21 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.45 +[2026-08-31 10:08:22] INFO: 174.1.60.1:46196 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:24 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.62 +[2026-08-31 10:08:24 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.64 +[2026-08-31 10:08:25] INFO: 174.1.60.1:47032 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:28 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.39 +[2026-08-31 10:08:28 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.30 +[2026-08-31 10:08:28] INFO: 174.1.60.1:47040 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:32 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.05 +[2026-08-31 10:08:32 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.11 +[2026-08-31 10:08:33] INFO: 174.1.60.1:39790 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:34 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 383.86 +[2026-08-31 10:08:34 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 383.73 +[2026-08-31 10:08:35] INFO: 174.1.60.1:39802 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:37 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.68 +[2026-08-31 10:08:37 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.57 +[2026-08-31 10:08:38] INFO: 174.1.60.1:39810 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:40 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 317.97 +[2026-08-31 10:08:40 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.31 +[2026-08-31 10:08:40] INFO: 174.1.60.1:39822 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:44 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.71 +[2026-08-31 10:08:44 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.66 +[2026-08-31 10:08:44] INFO: 174.1.60.1:37598 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:48 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.50 +[2026-08-31 10:08:48 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.47 +[2026-08-31 10:08:48] INFO: 174.1.60.1:37602 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:52 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.68 +[2026-08-31 10:08:52 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.59 +[2026-08-31 10:08:52] INFO: 174.1.60.1:49106 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:56 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.83 +[2026-08-31 10:08:56 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.85 +[2026-08-31 10:08:56] INFO: 174.1.60.1:49122 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:08:59 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.44 +[2026-08-31 10:08:59 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.45 +[2026-08-31 10:09:00] INFO: 174.1.60.1:49132 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:02 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.05 +[2026-08-31 10:09:02 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.07 +[2026-08-31 10:09:02] INFO: 174.1.60.1:58376 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:05 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.38 +[2026-08-31 10:09:05 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.40 +[2026-08-31 10:09:05] INFO: 174.1.60.1:58388 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:08 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.37 +[2026-08-31 10:09:08 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.48 +[2026-08-31 10:09:08] INFO: 174.1.60.1:58394 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:10 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.85 +[2026-08-31 10:09:10 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.93 +[2026-08-31 10:09:10] INFO: 174.1.60.1:58408 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:13 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.32 +[2026-08-31 10:09:13 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.23 +[2026-08-31 10:09:13] INFO: 174.1.60.1:39974 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:15 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.22 +[2026-08-31 10:09:15 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.20 +[2026-08-31 10:09:16] INFO: 174.1.60.1:39984 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:19 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 247.36 +[2026-08-31 10:09:19 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 247.30 +[2026-08-31 10:09:19] INFO: 174.1.60.1:39992 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:21 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.22 +[2026-08-31 10:09:21 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.25 +[2026-08-31 10:09:21] INFO: 174.1.60.1:39998 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:23 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.06 +[2026-08-31 10:09:23 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.25 +[2026-08-31 10:09:23] INFO: 174.1.60.1:47852 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:26 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.75 +[2026-08-31 10:09:26 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.85 +[2026-08-31 10:09:26] INFO: 174.1.60.1:47864 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:30 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.35 +[2026-08-31 10:09:30 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.36 +[2026-08-31 10:09:30] INFO: 174.1.60.1:47874 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:32 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 264.26 +[2026-08-31 10:09:32 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 264.22 +[2026-08-31 10:09:33] INFO: 174.1.60.1:37926 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:35 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.57 +[2026-08-31 10:09:35 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.52 +[2026-08-31 10:09:36] INFO: 174.1.60.1:37934 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:38 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.32 +[2026-08-31 10:09:38 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.23 +[2026-08-31 10:09:38] INFO: 174.1.60.1:37948 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:09:42 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.38 +[2026-08-31 10:09:42 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.37 +[2026-08-31 10:09:42] INFO: 174.1.60.1:40578 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:20] INFO: 174.1.60.1:39950 - "GET /model_info HTTP/1.1" 200 OK +[2026-08-31 10:10:21 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.44 +[2026-08-31 10:10:22 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.38 +[2026-08-31 10:10:22 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2641.16 +[2026-08-31 10:10:22 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5834.70 +[2026-08-31 10:10:22 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2639.05 +[2026-08-31 10:10:22 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 7775.66 +[2026-08-31 10:10:22 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5600.16 +[2026-08-31 10:10:22 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 7746.45 +[2026-08-31 10:10:22 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6002.11 +[2026-08-31 10:10:22 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5467.90 +[2026-08-31 10:10:22 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5046.48 +[2026-08-31 10:10:22 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5973.80 +[2026-08-31 10:10:22 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5773.97 +[2026-08-31 10:10:23 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4643.90 +[2026-08-31 10:10:23 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4770.35 +[2026-08-31 10:10:23 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 34617.72 +[2026-08-31 10:10:23] INFO: 174.1.60.1:39960 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:23] INFO: 174.1.60.1:39974 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:23] INFO: 174.1.60.1:39990 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:23] INFO: 174.1.60.1:39996 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:23] INFO: 174.1.60.1:40016 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:24] INFO: 174.1.60.1:40006 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:24] INFO: 174.1.60.1:40022 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:24] INFO: 174.1.60.1:40038 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:26 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 244.64 +[2026-08-31 10:10:26 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 244.85 +[2026-08-31 10:10:27] INFO: 174.1.60.1:58012 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:27 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 704.08 +[2026-08-31 10:10:27 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 706.08 +[2026-08-31 10:10:27 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2253.79 +[2026-08-31 10:10:27 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2235.97 +[2026-08-31 10:10:28] INFO: 174.1.60.1:58020 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:28 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 793.17 +[2026-08-31 10:10:28] INFO: 174.1.60.1:58036 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:28 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 722.04 +[2026-08-31 10:10:29 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1092.27 +[2026-08-31 10:10:29 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1270.21 +[2026-08-31 10:10:29] INFO: 174.1.60.1:58052 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:30 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 796.41 +[2026-08-31 10:10:30] INFO: 174.1.60.1:58054 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:30 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 793.56 +[2026-08-31 10:10:31] INFO: 174.1.60.1:58066 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:31 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 707.49 +[2026-08-31 10:10:31 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 709.25 +[2026-08-31 10:10:32] INFO: 174.1.60.1:58080 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:32 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 709.08 +[2026-08-31 10:10:32 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 710.42 +[2026-08-31 10:10:33] INFO: 174.1.60.1:56398 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:34 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 528.55 +[2026-08-31 10:10:34 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 527.52 +[2026-08-31 10:10:34] INFO: 174.1.60.1:56410 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:35 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 596.76 +[2026-08-31 10:10:35 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 596.78 +[2026-08-31 10:10:36 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 758.43 +[2026-08-31 10:10:36] INFO: 174.1.60.1:56414 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:36 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 681.21 +[2026-08-31 10:10:36 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1341.11 +[2026-08-31 10:10:37 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1324.99 +[2026-08-31 10:10:37] INFO: 174.1.60.1:56416 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:37 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 721.86 +[2026-08-31 10:10:37 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 805.30 +[2026-08-31 10:10:38] INFO: 174.1.60.1:56420 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:38] INFO: 174.1.60.1:56434 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:39 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 691.45 +[2026-08-31 10:10:39 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6163.97 +[2026-08-31 10:10:39 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 630.92 +[2026-08-31 10:10:39 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6038.46 +[2026-08-31 10:10:40 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 838.92 +[2026-08-31 10:10:40 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 957.74 +[2026-08-31 10:10:40] INFO: 174.1.60.1:56444 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:40] INFO: 174.1.60.1:56454 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:40] INFO: 174.1.60.1:56466 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:41 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 708.08 +[2026-08-31 10:10:41 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 709.85 +[2026-08-31 10:10:41] INFO: 174.1.60.1:56480 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:42 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 496.27 +[2026-08-31 10:10:42 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 495.35 +[2026-08-31 10:10:43] INFO: 174.1.60.1:47686 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:43 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 796.64 +[2026-08-31 10:10:43 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 796.54 +[2026-08-31 10:10:44] INFO: 174.1.60.1:47694 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:44 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 768.61 +[2026-08-31 10:10:44 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 699.55 +[2026-08-31 10:10:45 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1305.93 +[2026-08-31 10:10:45 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6494.83 +[2026-08-31 10:10:45 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1295.07 +[2026-08-31 10:10:45 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 34278.40 +[2026-08-31 10:10:45 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1561.85 +[2026-08-31 10:10:45] INFO: 174.1.60.1:47708 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:45 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1262.30 +[2026-08-31 10:10:46] INFO: 174.1.60.1:47720 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:46] INFO: 174.1.60.1:47728 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:46 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 859.40 +[2026-08-31 10:10:46 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 844.55 +[2026-08-31 10:10:46 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5227.02 +[2026-08-31 10:10:46 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6040.24 +[2026-08-31 10:10:47] INFO: 174.1.60.1:47740 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:47 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 785.42 +[2026-08-31 10:10:47 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 801.50 +[2026-08-31 10:10:47] INFO: 174.1.60.1:47742 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:48] INFO: 174.1.60.1:47752 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:48 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1143.62 +[2026-08-31 10:10:48 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1327.27 +[2026-08-31 10:10:48] INFO: 174.1.60.1:47762 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:49] INFO: 174.1.60.1:47776 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:49 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 716.83 +[2026-08-31 10:10:49 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 714.48 +[2026-08-31 10:10:49 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3129.52 +[2026-08-31 10:10:49 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3165.16 +[2026-08-31 10:10:50] INFO: 174.1.60.1:47778 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:50 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 811.14 +[2026-08-31 10:10:50] INFO: 174.1.60.1:47794 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:50 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 727.84 +[2026-08-31 10:10:51 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 903.31 +[2026-08-31 10:10:51 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 902.56 +[2026-08-31 10:10:51] INFO: 174.1.60.1:47800 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:52 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 723.76 +[2026-08-31 10:10:52 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 805.65 +[2026-08-31 10:10:52] INFO: 174.1.60.1:47812 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:53] INFO: 174.1.60.1:47820 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:53 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 756.54 +[2026-08-31 10:10:53 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 687.30 +[2026-08-31 10:10:53 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6788.12 +[2026-08-31 10:10:53 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6858.28 +[2026-08-31 10:10:53 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6518.63 +[2026-08-31 10:10:53 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 32932.21 +[2026-08-31 10:10:54 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2271.38 +[2026-08-31 10:10:54 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2320.91 +[2026-08-31 10:10:54] INFO: 174.1.60.1:34586 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:54] INFO: 174.1.60.1:34602 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:55] INFO: 174.1.60.1:34610 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:55] INFO: 174.1.60.1:34620 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:55 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 704.33 +[2026-08-31 10:10:55 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 704.79 +[2026-08-31 10:10:56] INFO: 174.1.60.1:34626 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:56 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 782.24 +[2026-08-31 10:10:56 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 781.00 +[2026-08-31 10:10:56] INFO: 174.1.60.1:34642 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:57 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 653.64 +[2026-08-31 10:10:57 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 655.14 +[2026-08-31 10:10:58] INFO: 174.1.60.1:34654 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:58 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 764.87 +[2026-08-31 10:10:58 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 764.21 +[2026-08-31 10:10:59] INFO: 174.1.60.1:34662 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:10:59 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 737.13 +[2026-08-31 10:10:59 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 663.08 +[2026-08-31 10:10:59 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1343.86 +[2026-08-31 10:10:59 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1663.01 +[2026-08-31 10:11:00] INFO: 174.1.60.1:34666 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:00] INFO: 174.1.60.1:34674 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:02 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 324.17 +[2026-08-31 10:11:02 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 323.51 +[2026-08-31 10:11:02] INFO: 174.1.60.1:34686 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:03 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 716.63 +[2026-08-31 10:11:03 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 717.18 +[2026-08-31 10:11:03] INFO: 174.1.60.1:49014 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:04 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 696.21 +[2026-08-31 10:11:04 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6878.28 +[2026-08-31 10:11:04 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 640.32 +[2026-08-31 10:11:04 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 26515.69 +[2026-08-31 10:11:05] INFO: 174.1.60.1:49026 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:05] INFO: 174.1.60.1:49036 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:05 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 672.76 +[2026-08-31 10:11:05 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 626.51 +[2026-08-31 10:11:05 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6717.64 +[2026-08-31 10:11:05 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 27448.20 +[2026-08-31 10:11:06] INFO: 174.1.60.1:49044 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:06 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 769.46 +[2026-08-31 10:11:06 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 701.42 +[2026-08-31 10:11:06 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6177.83 +[2026-08-31 10:11:06] INFO: 174.1.60.1:49058 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:06 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6395.61 +[2026-08-31 10:11:07 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1072.12 +[2026-08-31 10:11:07 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1233.22 +[2026-08-31 10:11:07] INFO: 174.1.60.1:49070 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:07 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1474.66 +[2026-08-31 10:11:07 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1502.14 +[2026-08-31 10:11:07] INFO: 174.1.60.1:49074 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:08] INFO: 174.1.60.1:49084 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:08 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 752.71 +[2026-08-31 10:11:08] INFO: 174.1.60.1:49094 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:08 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 752.00 +[2026-08-31 10:11:09 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3261.18 +[2026-08-31 10:11:09 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2276.86 +[2026-08-31 10:11:09 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 926.99 +[2026-08-31 10:11:09] INFO: 174.1.60.1:49108 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:09 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1058.25 +[2026-08-31 10:11:10] INFO: 174.1.60.1:49118 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:10] INFO: 174.1.60.1:49122 - "POST /generate HTTP/1.1" 200 OK +[2026-08-31 10:11:10 PP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 778.16 +[2026-08-31 10:11:10 PP1 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 782.65 +[2026-08-31 10:11:11] INFO: 174.1.60.1:49128 - "POST /generate HTTP/1.1" 200 OK diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_1.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_1.log new file mode 100644 index 0000000..b6dd790 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_1.log @@ -0,0 +1,962 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:41:36] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:41:36] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:41:36] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 09:41:39] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:41:39] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 1, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 831151606, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:39] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:05 PP2 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:05 PP2 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:08 PP3 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:08 PP3 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:08 PP2 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:08 PP2 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:09 PP2 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:09 PP2 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:09 PP2 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:09 PP2 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:09.977895 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:09.977967 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:09.977978 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:09.978009 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15025 +I0831 09:42:09.978089 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:10.002352 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:10.008647 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:10.015089 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:10.022050 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:10.024734 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:10.024832 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:10.024889 227 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:42:10 PP3 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:10 PP3 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0831 09:42:10.052026 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:10.053596 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:10.053627 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:10.079981 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:10.081375 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:10.081403 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:10.108022 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:10.109408 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:10.109434 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:10.116850 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:10.118253 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:10.118381 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:10 PP2 TP0 EP0] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:10 PP3 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:10 PP3 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:11 PP3 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:11 PP3 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:12.130067 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:12.130126 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:12.130137 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:12.130162 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15886 +I0831 09:42:12.130245 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:12.152841 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:12.159399 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:12.166726 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:12.174448 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:12.178123 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:12.178220 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:12.178277 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.186399 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:12.187916 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:12.187947 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.216012 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:12.217370 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:12.217394 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.244151 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:12.245579 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:12.245604 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.272070 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:12.273461 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:12.273599 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:12 PP3 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:42:12 PP2 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:12 PP2 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:12 PP2 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:12 PP2 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:12.667268 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:12.667344 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:12.667356 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:12.667390 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15407 +I0831 09:42:12.667484 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:12.671936 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:12.676013 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:12.679932 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:12.684485 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:12.686148 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:12.686215 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:12.686249 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.691023 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:12.692093 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:12.692114 230 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:42:12 PP3 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:12 PP3 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0831 09:42:12.715725 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:12.717128 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:12.717159 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.744067 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:12.745486 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:12.745520 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:12.772042 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:12.773432 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:12.773567 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:12 PP2 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:12 PP2 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:12 PP2 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:13.135846 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:13.135953 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:13.135968 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:13.136001 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16263 +I0831 09:42:13.136087 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:13.158528 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:13.165019 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:13.172000 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:13.178957 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:13.181638 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:13.181728 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:13.181782 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.188910 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:13.190310 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:13.190338 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.216014 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:13.217407 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:13.217433 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.244030 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:13.245443 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:13.245467 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.271930 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:13.273294 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:13.273419 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:13 PP2 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:42:13 PP3 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:13 PP3 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:13.738714 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:13.738770 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:13.738778 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:13.738801 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15512 +I0831 09:42:13.738865 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:13.761557 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:13.766111 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:13.771567 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:13.777050 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:13.779445 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:13.779529 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:13.779582 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.786026 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:13.787338 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:13.787364 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.812088 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:13.813455 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:13.813481 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.840025 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:13.841384 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:13.841408 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.868005 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:13.869371 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:13.869486 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:13 PP3 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:14 PP3 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:14 PP3 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:14.385574 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:14.385644 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:14.385653 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:14.385672 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15428 +I0831 09:42:14.385753 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:14.409547 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:14.414377 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:14.419886 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:14.425752 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:14.428140 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:14.428225 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:14.428275 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.434772 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:14.436095 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:14.436121 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.459998 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:14.461372 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:14.461398 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.488018 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:14.489419 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:14.489445 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:14.516022 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:14.517436 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:14.517565 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:14 PP3 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:42:16 PP2 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:16 PP2 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:17.060010 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:17.060086 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:17.060098 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:17.060127 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:15011 +I0831 09:42:17.060215 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:17.082297 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:17.088040 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:17.094589 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:17.101526 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:17.104231 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:17.104339 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:17.104398 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.111653 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:17.113193 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:17.113222 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.140111 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:17.141549 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:17.141577 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.168009 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:17.169421 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:17.169449 229 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:42:17 PP3 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:17 PP3 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +I0831 09:42:17.196063 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:17.197425 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:17.197567 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:17 PP2 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:17.372522 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:17.372584 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:17.372594 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.2 port: 12001 +I0831 09:42:17.372617 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.2:16508 +I0831 09:42:17.372694 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:17.394518 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:17.400167 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:17.407145 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:17.414136 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:17.416793 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:17.416887 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:17.416941 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.424130 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:17.425668 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:02 +I0831 09:42:17.425698 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.452132 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:17.453536 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:02 +I0831 09:42:17.453579 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.480042 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:17.481447 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:02 +I0831 09:42:17.481472 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.508090 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:17.509478 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:02 +I0831 09:42:17.509642 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:17 PP3 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:42:24 PP3 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:24 PP2 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP2 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP2 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP3 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:26 PP3 TP3 EP3] Init torch distributed ends. elapsed=12.57 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP3 TP2 EP2] Init torch distributed ends. elapsed=11.92 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP3 TP0 EP0] Init torch distributed ends. elapsed=14.17 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP2 TP3 EP3] Init torch distributed ends. elapsed=13.67 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP2 TP2 EP2] Init torch distributed ends. elapsed=9.24 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP2 TP1 EP1] Init torch distributed ends. elapsed=13.17 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP3 TP1 EP1] Init torch distributed ends. elapsed=8.93 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP2 TP0 EP0] Init torch distributed ends. elapsed=16.32 s, mem usage=0.58 GB +[2026-08-31 09:42:27 PP3 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP2 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP3 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP3 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP2 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP2 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP2 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP3 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP2 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP2 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP2 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP3 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP2 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP3 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP3 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP3 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP3 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP2 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP3 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP2 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP2 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP2 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP3 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP2 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP2 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP3 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP2 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP2 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP3 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP3 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP3 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP3 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:43:29 PP2 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:29 PP2 TP1 EP1] Load weight end. elapsed=62.75 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 09:43:30 PP2 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:30 PP2 TP2 EP2] Load weight end. elapsed=63.75 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP3 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:33 PP3 TP2 EP2] Load weight end. elapsed=66.29 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:33 PP2 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:33 PP2 TP3 EP3] Load weight end. elapsed=66.66 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 09:43:34 PP3 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:34 PP3 TP0 EP0] Load weight end. elapsed=67.13 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP2 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:35 PP2 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:35 PP2 TP0 EP0] Load weight end. elapsed=68.28 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=36.77 GB, mem usage=45.24 GB. +[2026-08-31 09:43:35 PP2 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:35 PP2 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:35 PP2 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:35 PP2 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:35 PP2 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:35 PP2 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:35 PP2 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:35 PP2 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:35 PP3 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:35 PP3 TP1 EP1] Load weight end. elapsed=68.81 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:36 PP3 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:36 PP3 TP3 EP3] Load weight end. elapsed=68.93 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:36 PP3 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP3 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP3 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP3 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP3 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP3 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP3 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP3 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP3 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:43:38 PP3 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP3 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP3 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:43:38 PP3 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP3 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP3 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:43:38 PP3 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP3 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP3 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:43:38 PP3 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP3 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:39 PP2 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:43:39 PP2 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP2 TP1 EP1] Load weight begin. avail mem=36.77 GB +[2026-08-31 09:43:39 PP2 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:43:39 PP2 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP2 TP3 EP3] Load weight begin. avail mem=36.77 GB +[2026-08-31 09:43:39 PP2 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:43:39 PP2 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP2 TP0 EP0] Load weight begin. avail mem=36.77 GB +[2026-08-31 09:43:39 PP2 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:43:39 PP2 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP2 TP2 EP2] Load weight begin. avail mem=36.77 GB +[2026-08-31 09:43:39 PP3 TP2 EP2] Load weight end. elapsed=0.75 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP3 TP3 EP3] Load weight end. elapsed=0.78 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP3 TP0 EP0] Load weight end. elapsed=0.82 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP3 TP1 EP1] Load weight end. elapsed=0.81 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP2 TP3 EP3] Load weight end. elapsed=0.73 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP2 TP1 EP1] Load weight end. elapsed=0.77 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 09:43:40 PP2 TP2 EP2] Load weight end. elapsed=0.74 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 09:43:40 PP2 TP0 EP0] Load weight end. elapsed=0.77 s, type=DFlashDraftModel, avail mem=34.65 GB, mem usage=2.12 GB. +[2026-08-31 09:43:40 PP2 TP0 EP0] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 09:43:40 PP2 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 09:43:40 PP3 TP2 EP2] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 09:43:40 PP2 TP2 EP2] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 09:43:40 PP2 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:40 PP2 TP1 EP1] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 09:43:40 PP3 TP0 EP0] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 09:43:40 PP3 TP3 EP3] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 09:43:40 PP2 TP3 EP3] DFLASH PP rank 2: capture layers=[], projection columns=[], incoming=True +[2026-08-31 09:43:40 PP3 TP1 EP1] DFLASH PP rank 3: capture layers=[37], projection columns=[1], incoming=True +[2026-08-31 09:43:40 PP3 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 09:43:40 PP3 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:41 PP3 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP3 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP3 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP3 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP2 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP2 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP2 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP2 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP3 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP3 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP3 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP2 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP3 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP2 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP2 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP2 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP3 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP3 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP3 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP3 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP2 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP2 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP2 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP2 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP2 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP2 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP2 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP2 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP3 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP3 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP3 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP3 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP3 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP3 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP3 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP3 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP2 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP2 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP2 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP2 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP2 TP3 EP3] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP2 TP2 EP2] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP2 TP0 EP0] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP2 TP1 EP1] Memory pool end. avail mem=28.66 GB +[2026-08-31 09:43:41 PP3 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP3 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP3 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP3 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP3 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP3 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP2 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP2 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP2 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP2 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp2_dp0.json +[2026-08-31 09:43:41 PP2 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp2_dp0.json +[2026-08-31 09:43:41 PP2 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp2_dp0.json +[2026-08-31 09:43:41 PP2 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp2_dp0.json +[2026-08-31 09:43:41 PP3 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP3 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp3_dp0.json +[2026-08-31 09:43:41 PP3 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp3_dp0.json +[2026-08-31 09:43:41 PP3 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp3_dp0.json +[2026-08-31 09:43:41 PP3 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp3_dp0.json +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:000, gen1: 310->0, gen2: 1074769->0 +[2026-08-31 09:45:28 PP2 TP2 EP2] Freezing GC in Scheduler process. gen0: 430->0, gen1: 310->0, gen2: 1074770->0 +[2026-08-31 09:45:28 PP2 TP0 EP0] Freezing GC in Scheduler process. gen0: 452->0, gen1: 310->0, gen2: 1075302->0 +[2026-08-31 09:45:29 PP2 TP3 EP3] Freezing GC in Scheduler process. gen0: 430->0, gen1: 310->0, gen2: 1074770->0 +[2026-08-31 09:45:29 PP3 TP2 EP2] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072831->0 +[2026-08-31 09:45:29 PP3 TP1 EP1] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072835->0 +[2026-08-31 09:45:29 PP3 TP0 EP0] Freezing GC in Scheduler process. gen0: 575->0, gen1: 3159->0, gen2: 1073191->0 +[2026-08-31 09:45:29 PP3 TP3 EP3] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072830->0 +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[2026-08-31 09:45:37 PP2 TP2 EP2] Triton kernel '_fwd_kernel' took 1.27 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:37 PP2 TP1 EP1] Triton kernel '_fwd_kernel' took 1.28 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:37 PP2 TP3 EP3] Triton kernel '_fwd_kernel' took 1.28 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:37 PP2 TP0 EP0] Triton kernel '_fwd_kernel' took 1.33 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:45 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:45 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:48 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.85 +[2026-08-31 09:45:48 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.86 +[2026-08-31 09:46:23 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:46:23 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:55:20 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP3 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP2 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP2 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP2 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP2 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP3 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP3 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP3 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 10:05:07 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:05:07 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:06:32 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:32 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:33 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 370.28 +[2026-08-31 10:06:34 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 370.84 +[2026-08-31 10:06:35 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 421.95 +[2026-08-31 10:06:35 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 422.15 +[2026-08-31 10:06:38 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 300.81 +[2026-08-31 10:06:38 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 300.29 +[2026-08-31 10:06:40 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 451.60 +[2026-08-31 10:06:40 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 452.82 +[2026-08-31 10:06:42 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 365.93 +[2026-08-31 10:06:42 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 365.13 +[2026-08-31 10:06:45 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.80 +[2026-08-31 10:06:45 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.81 +[2026-08-31 10:06:48 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.68 +[2026-08-31 10:06:48 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.92 +[2026-08-31 10:06:52 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.31 +[2026-08-31 10:06:52 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.07 +[2026-08-31 10:06:56 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.16 +[2026-08-31 10:06:56 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.22 +[2026-08-31 10:07:00 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 195.08 +[2026-08-31 10:07:00 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 195.22 +[2026-08-31 10:07:04 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 207.83 +[2026-08-31 10:07:04 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 207.70 +[2026-08-31 10:07:06 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.58 +[2026-08-31 10:07:07 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.81 +[2026-08-31 10:07:10 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.97 +[2026-08-31 10:07:10 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.83 +[2026-08-31 10:07:14 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.73 +[2026-08-31 10:07:14 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.76 +[2026-08-31 10:07:17 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.38 +[2026-08-31 10:07:17 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.39 +[2026-08-31 10:07:20 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.27 +[2026-08-31 10:07:20 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.27 +[2026-08-31 10:07:24 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 224.01 +[2026-08-31 10:07:24 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 223.97 +[2026-08-31 10:07:27 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.18 +[2026-08-31 10:07:27 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.20 +[2026-08-31 10:07:31 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.68 +[2026-08-31 10:07:31 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.66 +[2026-08-31 10:07:35 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.94 +[2026-08-31 10:07:35 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.93 +[2026-08-31 10:07:39 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 211.02 +[2026-08-31 10:07:39 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 211.09 +[2026-08-31 10:07:41 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.35 +[2026-08-31 10:07:41 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.22 +[2026-08-31 10:07:43 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.48 +[2026-08-31 10:07:43 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.56 +[2026-08-31 10:07:46 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.71 +[2026-08-31 10:07:46 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.70 +[2026-08-31 10:07:48 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 293.96 +[2026-08-31 10:07:48 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 293.95 +[2026-08-31 10:07:52 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.79 +[2026-08-31 10:07:52 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.68 +[2026-08-31 10:07:54 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 360.17 +[2026-08-31 10:07:54 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 360.36 +[2026-08-31 10:07:57 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.86 +[2026-08-31 10:07:57 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.80 +[2026-08-31 10:07:59 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 317.72 +[2026-08-31 10:07:59 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 317.77 +[2026-08-31 10:08:02 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.37 +[2026-08-31 10:08:02 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.36 +[2026-08-31 10:08:05 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.53 +[2026-08-31 10:08:05 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.43 +[2026-08-31 10:08:08 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 245.04 +[2026-08-31 10:08:08 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 245.01 +[2026-08-31 10:08:10 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 335.73 +[2026-08-31 10:08:10 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 335.94 +[2026-08-31 10:08:13 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.79 +[2026-08-31 10:08:13 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.77 +[2026-08-31 10:08:15 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 290.44 +[2026-08-31 10:08:15 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 290.54 +[2026-08-31 10:08:18 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.85 +[2026-08-31 10:08:18 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.90 +[2026-08-31 10:08:21 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.44 +[2026-08-31 10:08:21 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.25 +[2026-08-31 10:08:24 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.35 +[2026-08-31 10:08:24 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.39 +[2026-08-31 10:08:28 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.32 +[2026-08-31 10:08:28 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.49 +[2026-08-31 10:08:32 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.22 +[2026-08-31 10:08:32 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.15 +[2026-08-31 10:08:34 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 383.70 +[2026-08-31 10:08:35 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 384.16 +[2026-08-31 10:08:37 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.56 +[2026-08-31 10:08:37 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.32 +[2026-08-31 10:08:40 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.43 +[2026-08-31 10:08:40 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.39 +[2026-08-31 10:08:44 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.55 +[2026-08-31 10:08:44 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.63 +[2026-08-31 10:08:48 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.44 +[2026-08-31 10:08:48 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.35 +[2026-08-31 10:08:52 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.61 +[2026-08-31 10:08:52 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.66 +[2026-08-31 10:08:56 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.97 +[2026-08-31 10:08:56 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 199.02 +[2026-08-31 10:08:59 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.30 +[2026-08-31 10:08:59 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.27 +[2026-08-31 10:09:02 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.30 +[2026-08-31 10:09:02 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.39 +[2026-08-31 10:09:05 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.33 +[2026-08-31 10:09:05 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.31 +[2026-08-31 10:09:08 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.46 +[2026-08-31 10:09:08 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.57 +[2026-08-31 10:09:10 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.01 +[2026-08-31 10:09:10 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.87 +[2026-08-31 10:09:13 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.21 +[2026-08-31 10:09:13 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.29 +[2026-08-31 10:09:15 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.37 +[2026-08-31 10:09:15 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.41 +[2026-08-31 10:09:19 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 247.26 +[2026-08-31 10:09:19 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 247.01 +[2026-08-31 10:09:21 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.29 +[2026-08-31 10:09:21 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.78 +[2026-08-31 10:09:23 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.27 +[2026-08-31 10:09:23 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.30 +[2026-08-31 10:09:26 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.86 +[2026-08-31 10:09:26 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.93 +[2026-08-31 10:09:30 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.38 +[2026-08-31 10:09:30 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.35 +[2026-08-31 10:09:32 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 264.22 +[2026-08-31 10:09:32 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 264.27 +[2026-08-31 10:09:35 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.43 +[2026-08-31 10:09:35 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.46 +[2026-08-31 10:09:38 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.14 +[2026-08-31 10:09:38 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.10 +[2026-08-31 10:09:42 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.33 +[2026-08-31 10:09:42 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.26 +[2026-08-31 10:10:22 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.32 +[2026-08-31 10:10:22 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.27 +[2026-08-31 10:10:22 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2599.16 +[2026-08-31 10:10:22 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6275.98 +[2026-08-31 10:10:22 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2698.64 +[2026-08-31 10:10:22 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 31468.62 +[2026-08-31 10:10:22 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6219.50 +[2026-08-31 10:10:22 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 7016.54 +[2026-08-31 10:10:22 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 33786.16 +[2026-08-31 10:10:22 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5592.31 +[2026-08-31 10:10:22 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6203.79 +[2026-08-31 10:10:23 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4981.22 +[2026-08-31 10:10:23 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4959.69 +[2026-08-31 10:10:23 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 17357.75 +[2026-08-31 10:10:23 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 16771.57 +[2026-08-31 10:10:23 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 25531.66 +[2026-08-31 10:10:26 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 246.18 +[2026-08-31 10:10:26 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 246.99 +[2026-08-31 10:10:27 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 657.55 +[2026-08-31 10:10:27 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 612.46 +[2026-08-31 10:10:27 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2940.73 +[2026-08-31 10:10:27 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4404.45 +[2026-08-31 10:10:29 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 659.56 +[2026-08-31 10:10:29 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 613.82 +[2026-08-31 10:10:29 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1494.75 +[2026-08-31 10:10:29 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1811.78 +[2026-08-31 10:10:30 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 803.08 +[2026-08-31 10:10:30 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 800.30 +[2026-08-31 10:10:31 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 709.89 +[2026-08-31 10:10:31 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 712.21 +[2026-08-31 10:10:32 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 711.38 +[2026-08-31 10:10:32 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 712.27 +[2026-08-31 10:10:34 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 526.15 +[2026-08-31 10:10:34 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 524.63 +[2026-08-31 10:10:35 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 597.66 +[2026-08-31 10:10:35 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 598.65 +[2026-08-31 10:10:36 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 628.48 +[2026-08-31 10:10:36 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 580.78 +[2026-08-31 10:10:37 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1266.69 +[2026-08-31 10:10:37 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1223.74 +[2026-08-31 10:10:38 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 906.48 +[2026-08-31 10:10:38 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1041.08 +[2026-08-31 10:10:39 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 582.05 +[2026-08-31 10:10:39 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 538.04 +[2026-08-31 10:10:39 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5874.29 +[2026-08-31 10:10:39 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5503.37 +[2026-08-31 10:10:40 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1106.79 +[2026-08-31 10:10:40 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1328.71 +[2026-08-31 10:10:41 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 713.64 +[2026-08-31 10:10:41 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 719.20 +[2026-08-31 10:10:42 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 495.15 +[2026-08-31 10:10:42 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 494.55 +[2026-08-31 10:10:43 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 796.57 +[2026-08-31 10:10:43 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 795.35 +[2026-08-31 10:10:44 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 641.32 +[2026-08-31 10:10:45 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 595.16 +[2026-08-31 10:10:45 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1330.78 +[2026-08-31 10:10:45 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 46023.59 +[2026-08-31 10:10:45 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1335.01 +[2026-08-31 10:10:45 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 45852.76 +[2026-08-31 10:10:46 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1197.85 +[2026-08-31 10:10:46 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1158.96 +[2026-08-31 10:10:46 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 843.67 +[2026-08-31 10:10:47 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6136.26 +[2026-08-31 10:10:47 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 847.18 +[2026-08-31 10:10:47 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5750.05 +[2026-08-31 10:10:47 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 807.61 +[2026-08-31 10:10:48 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 836.67 +[2026-08-31 10:10:48 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1574.19 +[2026-08-31 10:10:48 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1912.94 +[2026-08-31 10:10:49 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 667.13 +[2026-08-31 10:10:49 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 607.76 +[2026-08-31 10:10:49 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4833.71 +[2026-08-31 10:10:49 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5343.49 +[2026-08-31 10:10:50 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 664.87 +[2026-08-31 10:10:51 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 658.22 +[2026-08-31 10:10:51 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 892.28 +[2026-08-31 10:10:51 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 886.92 +[2026-08-31 10:10:52 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 909.42 +[2026-08-31 10:10:52 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1040.29 +[2026-08-31 10:10:53 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 639.88 +[2026-08-31 10:10:53 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 22048.94 +[2026-08-31 10:10:53 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 591.26 +[2026-08-31 10:10:53 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 7576.33 +[2026-08-31 10:10:53 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 51797.03 +[2026-08-31 10:10:54 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6952.56 +[2026-08-31 10:10:54 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3059.13 +[2026-08-31 10:10:54 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4524.40 +[2026-08-31 10:10:55 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 704.26 +[2026-08-31 10:10:55 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 712.19 +[2026-08-31 10:10:56 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 780.86 +[2026-08-31 10:10:56 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 779.05 +[2026-08-31 10:10:57 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 655.62 +[2026-08-31 10:10:57 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 657.68 +[2026-08-31 10:10:58 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 764.80 +[2026-08-31 10:10:58 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 764.10 +[2026-08-31 10:10:59 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 611.77 +[2026-08-31 10:10:59 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 564.89 +[2026-08-31 10:11:00 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2074.47 +[2026-08-31 10:11:00 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2703.68 +[2026-08-31 10:11:02 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 323.11 +[2026-08-31 10:11:02 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 324.33 +[2026-08-31 10:11:03 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 716.89 +[2026-08-31 10:11:03 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 717.74 +[2026-08-31 10:11:04 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 636.25 +[2026-08-31 10:11:04 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 634.53 +[2026-08-31 10:11:04 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 34733.99 +[2026-08-31 10:11:04 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 32550.65 +[2026-08-31 10:11:05 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 622.92 +[2026-08-31 10:11:05 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 623.14 +[2026-08-31 10:11:05 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 33330.83 +[2026-08-31 10:11:05 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 7267.48 +[2026-08-31 10:11:06 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 643.84 +[2026-08-31 10:11:07 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 632.36 +[2026-08-31 10:11:07 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6048.53 +[2026-08-31 10:11:07 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6925.35 +[2026-08-31 10:11:07 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1285.21 +[2026-08-31 10:11:07 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1272.54 +[2026-08-31 10:11:08 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1797.78 +[2026-08-31 10:11:08 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1830.14 +[2026-08-31 10:11:09 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 692.96 +[2026-08-31 10:11:09 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 685.42 +[2026-08-31 10:11:09 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2185.02 +[2026-08-31 10:11:09 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2153.62 +[2026-08-31 10:11:10 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1220.26 +[2026-08-31 10:11:10 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1470.30 +[2026-08-31 10:11:11 PP2 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 786.06 +[2026-08-31 10:11:11 PP3 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 789.49 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_2.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_2.log new file mode 100644 index 0000000..544583c --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_2.log @@ -0,0 +1,963 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:41:39] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:41:39] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:41:39] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 09:41:42] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:41:42] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 2, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 494458008, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:42] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:11 PP5 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:11 PP5 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:13 PP4 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:13 PP4 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:13 PP5 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:13 PP5 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:14 PP5 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:14 PP5 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:15 PP5 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:15 PP5 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:15 PP4 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:15 PP4 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:15 PP5 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:15 PP5 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:16 PP4 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:16 PP4 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:16.098791 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:16.098867 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:16.098879 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:16.098910 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16292 +I0831 09:42:16.099007 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:16.123502 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:16.129731 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:16.136718 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:16.143671 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:16.146351 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:16.146467 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:16.146524 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.153684 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:16.155167 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:16.155200 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.180899 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:16.182284 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:16.182312 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.212934 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:16.214318 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:16.214347 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:16.244930 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:16.246331 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:16.246512 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:16 PP5 TP1 EP1] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:16 PP4 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:16 PP4 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:17 PP4 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:17 PP4 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:17 PP5 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:17 PP5 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:17.615634 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:17.615693 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:17.615702 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:17.615728 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15673 +I0831 09:42:17.615816 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:17.638916 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:17.646183 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:17.654594 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:17.663511 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:17.667169 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:17.667263 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:17.667317 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.675429 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:17.676898 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:17.676930 232 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:17.683689 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:17.683780 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:17.683799 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:17.683838 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15576 +I0831 09:42:17.683946 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:17.704919 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:17.706312 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:17.706336 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.707652 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:17.714234 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:17.720669 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:17.725912 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:17.727710 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:17.727766 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:17.727802 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.732877 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:17.733075 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:17.734097 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:17.734115 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.734288 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:17.734313 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.742641 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:17.744073 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:17.744196 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:17 PP5 TP0 EP0] Init torch distributed begin. +I0831 09:42:17.763118 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:17.764103 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:17.764122 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.786501 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:17.787505 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:17.787524 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:17.812850 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:17.814272 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:17.814422 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:17 PP4 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:42:18 PP5 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:18 PP5 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:18.227623 235 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:18.227677 235 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:18.227686 235 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:18.227708 235 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15431 +I0831 09:42:18.227774 235 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:18.249953 235 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:18.254477 235 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:18.258914 235 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:18.264343 235 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:18.266352 235 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:18.266431 235 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:18.266474 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.272147 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:18.273383 235 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:18.273407 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.296888 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:18.298244 235 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:18.298276 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.324867 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:18.326242 235 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:18.326290 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:18.352852 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:18.354280 235 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:18.354416 235 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:18 PP5 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:20 PP4 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:20 PP4 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:20.221573 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:20.221630 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:20.221640 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:20.221668 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15982 +I0831 09:42:20.221741 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:20.246462 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:20.251168 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:20.256633 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:20.262339 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:20.264744 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:20.264832 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:20.264883 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.271306 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:20.272641 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:20.272667 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.296852 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:20.298230 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:20.298257 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.324862 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:20.326228 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:20.326253 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.352880 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:20.354245 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:20.354380 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:20 PP4 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:42:20 PP5 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:20 PP5 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:20 PP4 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:20 PP4 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:20.715552 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:20.715633 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:20.715646 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:20.715677 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16770 +I0831 09:42:20.715809 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:20.740847 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:20.747913 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:20.754916 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:20.761780 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:20.764467 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:20.764559 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:20.764616 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.771657 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:20.773187 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:20.773216 234 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:20.785555 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:20.785631 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:20.785642 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:20.785669 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:16376 +I0831 09:42:20.785790 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:20.796865 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:20.798226 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:20.798255 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.812252 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:20.818806 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:20.824808 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:20.825702 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:20.826174 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:20.826198 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.832578 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:20.833449 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:20.834833 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:20.834954 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +I0831 09:42:20.835260 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:20.835348 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +[2026-08-31 09:42:20 PP5 TP2 EP2] Init torch distributed begin. +I0831 09:42:20.835402 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.860846 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:20.862295 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:20.862341 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.888852 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:20.890280 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:20.890326 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.897559 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:20.898983 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:20.899013 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.924901 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:20.926311 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:20.926458 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:20 PP4 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:21 PP4 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:21 PP4 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:21.523957 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:21.524031 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:21.524046 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.3 port: 12001 +I0831 09:42:21.524075 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.3:15743 +I0831 09:42:21.524171 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:21.528651 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:21.533006 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:21.537294 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:21.541604 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:21.543322 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:21.543395 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:21.543432 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:21.548393 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:21.549465 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:03 +I0831 09:42:21.549485 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:21.572810 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:21.574201 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:03 +I0831 09:42:21.574234 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:21.600970 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:21.602438 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:03 +I0831 09:42:21.602465 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:21.628973 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:21.630414 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:03 +I0831 09:42:21.630542 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:21 PP4 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:42:24 PP4 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:24 PP5 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP5 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP5 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP4 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:26 PP5 TP3 EP3] Init torch distributed ends. elapsed=8.05 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP5 TP2 EP2] Init torch distributed ends. elapsed=5.57 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP5 TP0 EP0] Init torch distributed ends. elapsed=8.66 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP4 TP3 EP3] Init torch distributed ends. elapsed=5.48 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP5 TP1 EP1] Init torch distributed ends. elapsed=10.16 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP4 TP1 EP1] Init torch distributed ends. elapsed=8.59 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP4 TP2 EP2] Init torch distributed ends. elapsed=4.77 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP4 TP0 EP0] Init torch distributed ends. elapsed=6.05 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP5 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:26 PP5 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:26 PP5 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:26 PP5 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP4 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP5 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP5 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP4 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP4 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP4 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP5 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP5 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP4 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP4 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP4 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP4 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP4 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP5 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP4 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP5 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP5 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP5 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP4 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP5 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP4 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP4 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP4 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP4 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP5 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP4 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP5 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP5 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:43:32 PP4 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:32 PP4 TP2 EP2] Load weight end. elapsed=65.03 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:32 PP5 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP5 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:33 PP5 TP1 EP1] Load weight end. elapsed=66.03 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP4 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP5 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:33 PP5 TP0 EP0] Load weight end. elapsed=66.83 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:34 PP5 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:34 PP5 TP2 EP2] Load weight end. elapsed=67.50 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:34 PP4 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:34 PP4 TP0 EP0] Load weight end. elapsed=67.54 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:35 PP5 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:35 PP5 TP3 EP3] Load weight end. elapsed=68.74 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:36 PP5 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP5 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP5 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP5 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP5 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP5 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP5 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:36 PP5 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:36 PP4 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:36 PP4 TP3 EP3] Load weight end. elapsed=69.53 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:37 PP4 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:37 PP4 TP1 EP1] Load weight end. elapsed=70.21 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:37 PP4 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:37 PP4 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:37 PP4 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:37 PP4 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:37 PP4 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:37 PP4 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:37 PP4 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:37 PP4 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP4 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:43:38 PP4 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP4 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP4 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:43:38 PP4 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP4 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP4 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:43:38 PP4 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP4 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP4 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:43:38 PP4 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP5 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:43:38 PP4 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP5 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:43:38 PP5 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP5 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP5 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP5 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP5 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:43:38 PP5 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP5 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:38 PP5 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:43:38 PP5 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP5 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:39 PP4 TP1 EP1] Load weight end. elapsed=0.99 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP4 TP0 EP0] Load weight end. elapsed=1.04 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP5 TP1 EP1] Load weight end. elapsed=0.99 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP5 TP0 EP0] Load weight end. elapsed=1.01 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP5 TP2 EP2] Load weight end. elapsed=1.04 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP4 TP2 EP2] Load weight end. elapsed=1.05 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP4 TP3 EP3] Load weight end. elapsed=1.08 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP5 TP3 EP3] Load weight end. elapsed=1.04 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-08-31 09:43:40 PP4 TP2 EP2] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 09:43:40 PP4 TP0 EP0] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 09:43:40 PP4 TP1 EP1] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 09:43:40 PP4 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 09:43:40 PP5 TP2 EP2] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 09:43:40 PP4 TP3 EP3] DFLASH PP rank 4: capture layers=[54], projection columns=[2], incoming=True +[2026-08-31 09:43:40 PP4 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:40 PP5 TP3 EP3] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 09:43:40 PP5 TP0 EP0] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 09:43:40 PP5 TP1 EP1] DFLASH PP rank 5: capture layers=[66], projection columns=[3], incoming=True +[2026-08-31 09:43:40 PP5 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 09:43:40 PP5 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:41 PP5 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP5 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP5 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP5 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP4 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP4 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP4 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP4 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP5 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP4 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP5 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP5 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP4 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP4 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP4 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP5 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP5 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP5 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP4 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP5 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP4 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP4 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP5 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP5 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP4 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP5 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP5 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP4 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP4 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP5 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP4 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP4 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP5 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP5 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP4 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP4 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP4 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP5 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP5 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP4 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp4_dp0.json +[2026-08-31 09:43:41 PP4 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp4_dp0.json +[2026-08-31 09:43:41 PP4 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp4_dp0.json +[2026-08-31 09:43:41 PP4 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp4_dp0.json +[2026-08-31 09:43:41 PP5 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP5 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp5_dp0.json +[2026-08-31 09:43:41 PP5 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp5_dp0.json +[2026-08-31 09:43:41 PP5 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp5_dp0.json +[2026-08-31 09:43:41 PP5 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp5_dp0.json + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:000, gen1: 310->0, gen2: 1075810->0 +[2026-08-31 09:45:29 PP4 TP0 EP0] Freezing GC in Scheduler process. gen0: 453->0, gen1: 310->0, gen2: 1076343->0 +[2026-08-31 09:45:29 PP4 TP3 EP3] Freezing GC in Scheduler process. gen0: 430->0, gen1: 310->0, gen2: 1075819->0 +[2026-08-31 09:45:29 PP4 TP2 EP2] Freezing GC in Scheduler process. gen0: 431->0, gen1: 310->0, gen2: 1075812->0 +[2026-08-31 09:45:29 PP5 TP2 EP2] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072834->0 +[2026-08-31 09:45:29 PP5 TP3 EP3] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072824->0 +[2026-08-31 09:45:30 PP5 TP0 EP0] Freezing GC in Scheduler process. gen0: 574->0, gen1: 3159->0, gen2: 1073200->0 +[2026-08-31 09:45:30 PP5 TP1 EP1] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072824->0 +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[2026-08-31 09:45:40 PP4 TP1 EP1] Triton kernel '_fwd_kernel' took 1.29 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:41 PP4 TP3 EP3] Triton kernel '_fwd_kernel' took 1.29 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:41 PP4 TP2 EP2] Triton kernel '_fwd_kernel' took 1.29 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:41 PP4 TP0 EP0] Triton kernel '_fwd_kernel' took 1.31 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:45 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:45 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:48 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.87 +[2026-08-31 09:45:48 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.89 +[2026-08-31 09:46:23 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:46:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:55:20 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP4 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP4 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP5 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP4 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP4 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP5 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP5 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP5 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 10:05:07 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:05:07 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:06:32 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:32 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:33 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 371.40 +[2026-08-31 10:06:33 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 371.34 +[2026-08-31 10:06:35 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 422.46 +[2026-08-31 10:06:35 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 423.21 +[2026-08-31 10:06:38 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 299.86 +[2026-08-31 10:06:38 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 299.52 +[2026-08-31 10:06:39 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 453.25 +[2026-08-31 10:06:39 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 453.45 +[2026-08-31 10:06:42 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 364.82 +[2026-08-31 10:06:42 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 364.83 +[2026-08-31 10:06:45 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.86 +[2026-08-31 10:06:45 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.80 +[2026-08-31 10:06:48 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.90 +[2026-08-31 10:06:48 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.99 +[2026-08-31 10:06:52 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.01 +[2026-08-31 10:06:52 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 203.98 +[2026-08-31 10:06:56 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.20 +[2026-08-31 10:06:56 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.20 +[2026-08-31 10:07:00 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 195.28 +[2026-08-31 10:07:00 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 195.23 +[2026-08-31 10:07:04 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 207.69 +[2026-08-31 10:07:04 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 207.75 +[2026-08-31 10:07:06 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.81 +[2026-08-31 10:07:06 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.73 +[2026-08-31 10:07:10 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.90 +[2026-08-31 10:07:10 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.97 +[2026-08-31 10:07:14 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.73 +[2026-08-31 10:07:14 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.58 +[2026-08-31 10:07:17 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.39 +[2026-08-31 10:07:17 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.54 +[2026-08-31 10:07:20 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.22 +[2026-08-31 10:07:20 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.14 +[2026-08-31 10:07:24 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 224.11 +[2026-08-31 10:07:24 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 224.08 +[2026-08-31 10:07:27 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.14 +[2026-08-31 10:07:27 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.14 +[2026-08-31 10:07:31 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.68 +[2026-08-31 10:07:31 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.78 +[2026-08-31 10:07:35 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.96 +[2026-08-31 10:07:35 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.96 +[2026-08-31 10:07:39 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 211.06 +[2026-08-31 10:07:39 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 210.93 +[2026-08-31 10:07:41 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.23 +[2026-08-31 10:07:41 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.50 +[2026-08-31 10:07:43 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.43 +[2026-08-31 10:07:43 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.39 +[2026-08-31 10:07:46 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.85 +[2026-08-31 10:07:46 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.88 +[2026-08-31 10:07:48 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 293.90 +[2026-08-31 10:07:48 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 293.93 +[2026-08-31 10:07:52 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.46 +[2026-08-31 10:07:52 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.41 +[2026-08-31 10:07:54 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 360.87 +[2026-08-31 10:07:54 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 360.86 +[2026-08-31 10:07:57 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.72 +[2026-08-31 10:07:57 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.64 +[2026-08-31 10:07:59 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 317.88 +[2026-08-31 10:07:59 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.21 +[2026-08-31 10:08:02 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.22 +[2026-08-31 10:08:02 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.07 +[2026-08-31 10:08:05 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.39 +[2026-08-31 10:08:05 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.37 +[2026-08-31 10:08:08 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 244.93 +[2026-08-31 10:08:08 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 244.95 +[2026-08-31 10:08:10 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 336.06 +[2026-08-31 10:08:10 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 336.06 +[2026-08-31 10:08:13 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.71 +[2026-08-31 10:08:13 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.74 +[2026-08-31 10:08:15 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 290.54 +[2026-08-31 10:08:15 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 290.25 +[2026-08-31 10:08:18 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.99 +[2026-08-31 10:08:18 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 330.30 +[2026-08-31 10:08:21 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.24 +[2026-08-31 10:08:21 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.11 +[2026-08-31 10:08:24 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.25 +[2026-08-31 10:08:24 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.70 +[2026-08-31 10:08:28 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.48 +[2026-08-31 10:08:28 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.29 +[2026-08-31 10:08:32 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.14 +[2026-08-31 10:08:32 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.13 +[2026-08-31 10:08:34 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 384.21 +[2026-08-31 10:08:34 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 384.45 +[2026-08-31 10:08:37 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.34 +[2026-08-31 10:08:37 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.30 +[2026-08-31 10:08:40 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.31 +[2026-08-31 10:08:40 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.27 +[2026-08-31 10:08:44 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.61 +[2026-08-31 10:08:44 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.71 +[2026-08-31 10:08:48 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.32 +[2026-08-31 10:08:48 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.24 +[2026-08-31 10:08:52 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.75 +[2026-08-31 10:08:52 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.79 +[2026-08-31 10:08:56 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.92 +[2026-08-31 10:08:56 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.95 +[2026-08-31 10:08:59 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.24 +[2026-08-31 10:08:59 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.02 +[2026-08-31 10:09:02 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.47 +[2026-08-31 10:09:02 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.69 +[2026-08-31 10:09:05 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.27 +[2026-08-31 10:09:05 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.34 +[2026-08-31 10:09:08 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.58 +[2026-08-31 10:09:08 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.59 +[2026-08-31 10:09:10 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.86 +[2026-08-31 10:09:10 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.85 +[2026-08-31 10:09:13 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.29 +[2026-08-31 10:09:13 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.35 +[2026-08-31 10:09:15 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.24 +[2026-08-31 10:09:15 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.28 +[2026-08-31 10:09:19 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 247.05 +[2026-08-31 10:09:19 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 246.98 +[2026-08-31 10:09:21 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.77 +[2026-08-31 10:09:21 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.87 +[2026-08-31 10:09:23 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.27 +[2026-08-31 10:09:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.31 +[2026-08-31 10:09:26 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.83 +[2026-08-31 10:09:26 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.57 +[2026-08-31 10:09:30 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.44 +[2026-08-31 10:09:30 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.62 +[2026-08-31 10:09:32 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 264.11 +[2026-08-31 10:09:32 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 263.68 +[2026-08-31 10:09:35 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.41 +[2026-08-31 10:09:35 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.64 +[2026-08-31 10:09:38 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.14 +[2026-08-31 10:09:38 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 302.86 +[2026-08-31 10:09:42 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.13 +[2026-08-31 10:09:42 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.10 +[2026-08-31 10:10:22 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.21 +[2026-08-31 10:10:22 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.16 +[2026-08-31 10:10:22 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2862.15 +[2026-08-31 10:10:22 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6156.66 +[2026-08-31 10:10:22 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2826.17 +[2026-08-31 10:10:22 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 21964.91 +[2026-08-31 10:10:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4822.38 +[2026-08-31 10:10:23 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5128.65 +[2026-08-31 10:10:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 17782.60 +[2026-08-31 10:10:23 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 21408.58 +[2026-08-31 10:10:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 25520.81 +[2026-08-31 10:10:23 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 25401.91 +[2026-08-31 10:10:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 18573.09 +[2026-08-31 10:10:23 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19484.09 +[2026-08-31 10:10:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 26728.24 +[2026-08-31 10:10:23 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 32787.88 +[2026-08-31 10:10:26 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.25 +[2026-08-31 10:10:26 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.43 +[2026-08-31 10:10:27 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 568.55 +[2026-08-31 10:10:27 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 569.93 +[2026-08-31 10:10:28 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4783.92 +[2026-08-31 10:10:28 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2813.88 +[2026-08-31 10:10:29 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 607.83 +[2026-08-31 10:10:29 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 613.86 +[2026-08-31 10:10:29 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1839.75 +[2026-08-31 10:10:29 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1889.92 +[2026-08-31 10:10:30 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 881.72 +[2026-08-31 10:10:30 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 981.78 +[2026-08-31 10:10:31 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 713.99 +[2026-08-31 10:10:31 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 716.17 +[2026-08-31 10:10:32 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 713.93 +[2026-08-31 10:10:32 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 715.70 +[2026-08-31 10:10:34 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 523.14 +[2026-08-31 10:10:34 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 521.64 +[2026-08-31 10:10:35 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 565.96 +[2026-08-31 10:10:35 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 529.85 +[2026-08-31 10:10:36 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 573.10 +[2026-08-31 10:10:36 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 569.75 +[2026-08-31 10:10:37 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1179.76 +[2026-08-31 10:10:37 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1180.47 +[2026-08-31 10:10:38 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1239.63 +[2026-08-31 10:10:38 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1480.32 +[2026-08-31 10:10:39 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 495.17 +[2026-08-31 10:10:39 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6652.15 +[2026-08-31 10:10:39 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 465.38 +[2026-08-31 10:10:39 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6105.50 +[2026-08-31 10:10:40 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1580.76 +[2026-08-31 10:10:40 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2038.93 +[2026-08-31 10:10:41 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 725.21 +[2026-08-31 10:10:41 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 726.94 +[2026-08-31 10:10:42 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 493.48 +[2026-08-31 10:10:42 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 468.33 +[2026-08-31 10:10:43 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 795.63 +[2026-08-31 10:10:43 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 792.40 +[2026-08-31 10:10:45 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 555.63 +[2026-08-31 10:10:45 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 545.61 +[2026-08-31 10:10:45 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1334.66 +[2026-08-31 10:10:45 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 40866.63 +[2026-08-31 10:10:45 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1353.29 +[2026-08-31 10:10:45 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 54469.65 +[2026-08-31 10:10:46 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1120.74 +[2026-08-31 10:10:46 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1109.63 +[2026-08-31 10:10:47 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 846.21 +[2026-08-31 10:10:47 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6554.43 +[2026-08-31 10:10:47 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 849.71 +[2026-08-31 10:10:47 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6189.55 +[2026-08-31 10:10:48 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 844.04 +[2026-08-31 10:10:48 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 859.44 +[2026-08-31 10:10:48 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2485.96 +[2026-08-31 10:10:48 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3336.62 +[2026-08-31 10:10:49 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 606.38 +[2026-08-31 10:10:49 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 565.52 +[2026-08-31 10:10:49 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3201.51 +[2026-08-31 10:10:50 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3283.51 +[2026-08-31 10:10:51 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 662.50 +[2026-08-31 10:10:51 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 661.49 +[2026-08-31 10:10:52 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 858.62 +[2026-08-31 10:10:52 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 857.94 +[2026-08-31 10:10:52 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1078.24 +[2026-08-31 10:10:52 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1084.47 +[2026-08-31 10:10:54 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 592.38 +[2026-08-31 10:10:54 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 47888.99 +[2026-08-31 10:10:54 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 591.91 +[2026-08-31 10:10:54 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 54235.11 +[2026-08-31 10:10:54 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6700.20 +[2026-08-31 10:10:54 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 36528.47 +[2026-08-31 10:10:54 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9345.73 +[2026-08-31 10:10:54 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6041.01 +[2026-08-31 10:10:55 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 717.87 +[2026-08-31 10:10:55 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 694.75 +[2026-08-31 10:10:56 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 777.88 +[2026-08-31 10:10:56 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 848.90 +[2026-08-31 10:10:57 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 656.37 +[2026-08-31 10:10:57 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 656.10 +[2026-08-31 10:10:58 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 766.35 +[2026-08-31 10:10:58 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 709.80 +[2026-08-31 10:10:59 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 527.08 +[2026-08-31 10:10:59 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 520.54 +[2026-08-31 10:11:00 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4019.86 +[2026-08-31 10:11:00 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 8423.87 +[2026-08-31 10:11:02 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 323.84 +[2026-08-31 10:11:02 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 323.42 +[2026-08-31 10:11:03 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 717.28 +[2026-08-31 10:11:03 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 718.01 +[2026-08-31 10:11:04 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 633.33 +[2026-08-31 10:11:04 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 36076.68 +[2026-08-31 10:11:04 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 631.29 +[2026-08-31 10:11:04 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 29201.75 +[2026-08-31 10:11:05 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 582.44 +[2026-08-31 10:11:05 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 544.20 +[2026-08-31 10:11:05 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6379.34 +[2026-08-31 10:11:06 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6465.71 +[2026-08-31 10:11:07 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 637.03 +[2026-08-31 10:11:07 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 636.56 +[2026-08-31 10:11:07 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6478.39 +[2026-08-31 10:11:07 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6750.80 +[2026-08-31 10:11:07 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1261.32 +[2026-08-31 10:11:07 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1257.13 +[2026-08-31 10:11:08 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1836.31 +[2026-08-31 10:11:08 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1892.52 +[2026-08-31 10:11:09 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 686.09 +[2026-08-31 10:11:09 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 676.29 +[2026-08-31 10:11:09 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2247.29 +[2026-08-31 10:11:09 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2269.52 +[2026-08-31 10:11:10 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1764.77 +[2026-08-31 10:11:10 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1740.62 +[2026-08-31 10:11:11 PP4 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 792.15 +[2026-08-31 10:11:11 PP5 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 889.86 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_3.log new file mode 100644 index 0000000..da73a23 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/p_3.log @@ -0,0 +1,972 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-08-31 09:41:41] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-08-31 09:41:41] Attention backend not specified. Use triton backend by default. +[2026-08-31 09:41:41] Pipeline parallelism is incompatible with overlap schedule. +[2026-08-31 09:41:45] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-08-31 09:41:45] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.88, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 3, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 62676149, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-08-31 09:41:45] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:09 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:09 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:13 PP6 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:13 PP6 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:13.695473 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:13.695561 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:13.695572 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:13.695604 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16501 +I0831 09:42:13.695696 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:13.717430 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:13.723091 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:13.729344 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:13.736286 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:13.738999 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:13.739102 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:13.739153 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.746323 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:13.747885 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:13.747915 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.771523 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:13.772876 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:13.772902 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.799546 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:13.800921 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:13.800945 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:13.827522 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:13.828886 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:13.829025 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:13 PP6 TP0 EP0] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:15 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:15 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:17 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:17 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:17 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:17 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:17 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:17 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-08-31 09:42:18 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:18 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:18 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:18 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:42:19 PP6 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:19 PP6 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:19.231510 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:19.231586 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:19.231597 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:19.231652 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16470 +I0831 09:42:19.231736 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:19.257901 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:19.266362 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:19.275331 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:19.284246 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:19.287925 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:19.288017 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:19.288074 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:19.296166 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:19.297622 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:19.297695 228 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-08-31 09:42:19 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:42:19 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0831 09:42:19.323586 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:19.324963 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:19.325001 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:19.351558 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:19.352977 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:19.353027 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:19.379528 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:19.380888 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:19.381045 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:19 PP6 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:42:20 PP7 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:20 PP7 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:20 PP7 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:20 PP7 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:20.739475 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:20.739539 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:20.739550 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:20.739576 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16766 +I0831 09:42:20.739660 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:20.763628 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:20.770818 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:20.777843 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:20.784749 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:20.787426 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:20.787525 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:20.787585 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.794713 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:20.796140 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:20.796168 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.823416 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:20.824784 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:20.824843 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.851555 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:20.852942 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:20.852965 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.879483 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:20.880846 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:20.880972 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:20 PP7 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:20.888545 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:20.888597 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:20.888607 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:20.888629 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16974 +I0831 09:42:20.888702 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:20.912449 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:20.916940 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:20.921387 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:20.926817 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:20.928807 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:20.928880 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:20.928921 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.934689 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:20.935930 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:20.935971 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.959431 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:20.960767 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:20.960803 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:20.987465 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:20.988829 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:20.988853 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:21.015460 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:21.016796 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:21.016921 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:21 PP7 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:42:21 PP7 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:21 PP7 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:21.998988 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:21.999050 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:21.999059 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:21.999082 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15019 +I0831 09:42:21.999173 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:22.021488 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:22.027136 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:22.033439 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:22.040371 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:22.043017 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:22.043107 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:22.043160 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:22.050271 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:22.051681 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:22.051709 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:22.075457 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:22.076792 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:22.076820 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:22.103514 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:22.104875 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:22.104903 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:22.131475 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:22.132820 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:22.132942 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:22 PP7 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:22 PP6 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:22 PP6 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-08-31 09:42:22 PP7 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:22 PP7 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:23.179744 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:23.179831 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:23.179842 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:23.179872 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15326 +I0831 09:42:23.180012 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:23.203395 229 topology.cpp:127] Device mlx5_0 port 1 is available +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:23.207616 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:23.207685 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:23.207696 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:23.207726 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16307 +I0831 09:42:23.207810 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:23.210486 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:23.217447 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:23.224363 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:23.227010 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:23.227104 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:23.227159 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.228972 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:23.234318 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:23.234490 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:23.235746 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:23.235777 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.259354 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:23.259539 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:23.260876 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:23.260900 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.266358 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:23.269038 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:23.269129 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:23.269187 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.276319 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:23.277894 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:23.277923 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.287528 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:23.288887 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:23.288910 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.303479 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:23.304854 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:23.304898 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.315529 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:23.316870 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:23.317019 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:23 PP6 TP2 EP2] Init torch distributed begin. +I0831 09:42:23.331475 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:23.332870 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:23.332894 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:23.359462 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:23.360816 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:23.360947 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:23 PP7 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:42:23 PP6 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-08-31 09:42:23 PP6 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0831 09:42:24.196678 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0831 09:42:24.196749 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0831 09:42:24.196759 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0831 09:42:24.196784 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16294 +I0831 09:42:24.196864 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0831 09:42:24.222370 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0831 09:42:24.228900 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0831 09:42:24.235908 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0831 09:42:24.242949 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0831 09:42:24.245643 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0831 09:42:24.245740 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0831 09:42:24.245798 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:24.252971 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0831 09:42:24.254403 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0831 09:42:24.254477 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:24.261885 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0831 09:42:24.263235 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0831 09:42:24.263260 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:24.270509 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0831 09:42:24.271867 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0831 09:42:24.271891 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0831 09:42:24.295508 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0831 09:42:24.296914 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0831 09:42:24.297055 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-08-31 09:42:24 PP6 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:42:24 PP7 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:24 PP6 TP0 EP0] sglang is using nccl==2.30.7 +[2026-08-31 09:42:25 PP6 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP6 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:25 PP7 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-08-31 09:42:26 PP7 TP3 EP3] Init torch distributed ends. elapsed=4.25 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP7 TP2 EP2] Init torch distributed ends. elapsed=5.50 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP7 TP1 EP1] Init torch distributed ends. elapsed=5.37 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP7 TP0 EP0] Init torch distributed ends. elapsed=3.02 s, mem usage=0.50 GB +[2026-08-31 09:42:26 PP6 TP3 EP3] Init torch distributed ends. elapsed=2.09 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP6 TP2 EP2] Init torch distributed ends. elapsed=3.07 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP6 TP1 EP1] Init torch distributed ends. elapsed=7.00 s, mem usage=0.58 GB +[2026-08-31 09:42:26 PP6 TP0 EP0] Init torch distributed ends. elapsed=12.56 s, mem usage=0.58 GB +[2026-08-31 09:42:27 PP6 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP6 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP7 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP7 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP6 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP6 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP6 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP7 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP7 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP6 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP7 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP7 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-08-31 09:42:27 PP6 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-08-31 09:42:27 PP7 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP7 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP6 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-08-31 09:42:27 PP6 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP6 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP7 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP7 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP6 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP6 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP7 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP6 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP7 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP6 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP7 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP7 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP7 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP7 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:42:27 PP6 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-08-31 09:42:27 PP6 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:31 PP6 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:31 PP6 TP3 EP3] Load weight end. elapsed=64.40 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:32 PP7 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:32 PP7 TP0 EP0] Load weight end. elapsed=65.11 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:33 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-08-31 09:43:34 PP7 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:34 PP7 TP1 EP1] Load weight end. elapsed=67.02 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 09:43:34 PP6 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:34 PP6 TP2 EP2] Load weight end. elapsed=67.21 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:34 PP6 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:34 PP6 TP0 EP0] Load weight end. elapsed=67.31 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:35 PP7 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:35 PP7 TP2 EP2] Load weight end. elapsed=68.73 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 09:43:37 PP7 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:37 PP7 TP3 EP3] Load weight end. elapsed=70.79 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-08-31 09:43:38 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP7 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:43:38 PP7 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP7 TP0 EP0] Load weight begin. avail mem=32.33 GB +[2026-08-31 09:43:38 PP7 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:43:38 PP7 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP7 TP3 EP3] Load weight begin. avail mem=32.33 GB +[2026-08-31 09:43:38 PP7 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:43:38 PP7 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP7 TP1 EP1] Load weight begin. avail mem=32.33 GB +[2026-08-31 09:43:38 PP7 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:43:38 PP7 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:38 PP7 TP2 EP2] Load weight begin. avail mem=32.33 GB +[2026-08-31 09:43:38 PP6 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-08-31 09:43:38 PP6 TP1 EP1] Load weight end. elapsed=71.45 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-08-31 09:43:38 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-08-31 09:43:38 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:38 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-08-31 09:43:39 PP7 TP0 EP0] Load weight end. elapsed=0.68 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 09:43:39 PP6 TP3 EP3] Init torch distributed begin. +[2026-08-31 09:43:39 PP6 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP6 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:39 PP6 TP1 EP1] Init torch distributed begin. +[2026-08-31 09:43:39 PP6 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP6 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:39 PP6 TP0 EP0] Init torch distributed begin. +[2026-08-31 09:43:39 PP6 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP6 TP2 EP2] Init torch distributed begin. +[2026-08-31 09:43:39 PP6 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:39 PP6 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-08-31 09:43:39 PP6 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-08-31 09:43:39 PP7 TP3 EP3] Load weight end. elapsed=0.72 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 09:43:39 PP7 TP1 EP1] Load weight end. elapsed=0.71 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 09:43:39 PP7 TP2 EP2] Load weight end. elapsed=0.69 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-08-31 09:43:39 PP6 TP2 EP2] Load weight end. elapsed=0.71 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP6 TP0 EP0] Load weight end. elapsed=0.71 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP6 TP1 EP1] Load weight end. elapsed=0.77 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:39 PP6 TP3 EP3] Load weight end. elapsed=0.79 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-08-31 09:43:40 PP6 TP0 EP0] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 09:43:40 PP6 TP2 EP2] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 09:43:40 PP7 TP2 EP2] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 09:43:40 PP6 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 09:43:40 PP7 TP0 EP0] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 09:43:40 PP6 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:40 PP7 TP1 EP1] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 09:43:40 PP7 TP3 EP3] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-08-31 09:43:40 PP6 TP3 EP3] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 09:43:40 PP6 TP1 EP1] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-08-31 09:43:40 PP7 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-08-31 09:43:40 PP7 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-08-31 09:43:41 PP7 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP7 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP7 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP7 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP6 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP6 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP6 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP6 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-08-31 09:43:41 PP7 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP7 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP7 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP6 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP7 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-08-31 09:43:41 PP6 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP6 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP6 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-08-31 09:43:41 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP6 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP6 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 09:43:41 PP7 TP1 EP1] Memory pool end. avail mem=22.26 GB +[2026-08-31 09:43:41 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 09:43:41 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 09:43:41 PP6 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 7.79 GB +[2026-08-31 09:43:41 PP7 TP2 EP2] Memory pool end. avail mem=22.26 GB +[2026-08-31 09:43:41 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, KV size: 5.84 GB +[2026-08-31 09:43:41 PP7 TP0 EP0] Memory pool end. avail mem=22.26 GB +[2026-08-31 09:43:41 PP7 TP3 EP3] Memory pool end. avail mem=22.26 GB +[2026-08-31 09:43:41 PP6 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP6 TP0 EP0] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP6 TP3 EP3] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP6 TP2 EP2] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-08-31 09:43:41 PP6 TP1 EP1] Memory pool end. avail mem=24.59 GB +[2026-08-31 09:43:41 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 09:43:41 PP7 TP1 EP1] Memory pool end. avail mem=11.88 GB +[2026-08-31 09:43:41 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 09:43:41 PP7 TP2 EP2] Memory pool end. avail mem=11.88 GB +[2026-08-31 09:43:41 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 09:43:41 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1814144, K size: 5.19 GB, V size: 5.19 GB +[2026-08-31 09:43:41 PP7 TP3 EP3] Memory pool end. avail mem=11.88 GB +[2026-08-31 09:43:41 PP7 TP0 EP0] Memory pool end. avail mem=11.88 GB +[2026-08-31 09:43:41 PP7 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP7 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP7 TP0 EP0] Overriding draft attention backend to triton. +[2026-08-31 09:43:41 PP6 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-08-31 09:43:41 PP6 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-08-31 09:43:41 PP7 TP1 EP1] Overriding draft attention backend to triton. +[2026-08-31 09:43:41 PP7 TP3 EP3] Overriding draft attention backend to triton. +[2026-08-31 09:43:41 PP7 TP2 EP2] Overriding draft attention backend to triton. +[2026-08-31 09:43:41 PP6 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP6 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp6_dp0.json +[2026-08-31 09:43:41 PP6 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp6_dp0.json +[2026-08-31 09:43:41 PP6 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp6_dp0.json +[2026-08-31 09:43:41 PP6 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp6_dp0.json +[2026-08-31 09:43:41 PP7 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-08-31 09:43:41 PP7 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp7_dp0.json +[2026-08-31 09:43:41 PP7 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp7_dp0.json +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-08-31 09:43:41 PP7 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp7_dp0.json +[2026-08-31 09:43:41 PP7 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp7_dp0.json +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:000, gen1: 310->0, gen2: 1075813->0 +[2026-08-31 09:45:30 PP6 TP1 EP1] Freezing GC in Scheduler process. gen0: 431->0, gen1: 310->0, gen2: 1075819->0 +[2026-08-31 09:45:30 PP6 TP2 EP2] Freezing GC in Scheduler process. gen0: 431->0, gen1: 310->0, gen2: 1075810->0 +[2026-08-31 09:45:30 PP6 TP0 EP0] Freezing GC in Scheduler process. gen0: 453->0, gen1: 310->0, gen2: 1076346->0 +[2026-08-31 09:45:30 PP7 TP2 EP2] Freezing GC in Scheduler process. gen0: 307->0, gen1: 0->0, gen2: 1076106->0 +[2026-08-31 09:45:30 PP7 TP1 EP1] Freezing GC in Scheduler process. gen0: 307->0, gen1: 0->0, gen2: 1076110->0 +[2026-08-31 09:45:30 PP7 TP3 EP3] Freezing GC in Scheduler process. gen0: 307->0, gen1: 0->0, gen2: 1076107->0 +[2026-08-31 09:45:30 PP7 TP0 EP0] Freezing GC in Scheduler process. gen0: 321->0, gen1: 24->0, gen2: 1076615->0 +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +/opt/sglang/lib/python3.12/site-packages/triton/language/core.py:2284: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. + warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +[2026-08-31 09:45:44 PP6 TP0 EP0] Triton kernel '_fwd_kernel' took 1.15 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:44 PP6 TP3 EP3] Triton kernel '_fwd_kernel' took 1.18 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:44 PP6 TP1 EP1] Triton kernel '_fwd_kernel' took 1.18 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +[2026-08-31 09:45:44 PP6 TP2 EP2] Triton kernel '_fwd_kernel' took 1.18 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. +/opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead. + return func(*args, **kwargs) +[2026-08-31 09:45:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3.66 +[2026-08-31 09:45:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.90 +[2026-08-31 09:45:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20.90 +[2026-08-31 09:46:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:46:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.81 +[2026-08-31 09:55:20 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.12 +[2026-08-31 09:55:20 PP6 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP6 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP6 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP7 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP6 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP7 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP7 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 09:55:20 PP7 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-08-31 10:05:07 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:05:07 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.11 +[2026-08-31 10:06:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9.04 +[2026-08-31 10:06:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 371.56 +[2026-08-31 10:06:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 372.18 +[2026-08-31 10:06:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 423.87 +[2026-08-31 10:06:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 424.35 +[2026-08-31 10:06:38 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 299.15 +[2026-08-31 10:06:38 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 298.87 +[2026-08-31 10:06:39 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 454.50 +[2026-08-31 10:06:40 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 455.08 +[2026-08-31 10:06:42 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 364.22 +[2026-08-31 10:06:42 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 364.33 +[2026-08-31 10:06:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.79 +[2026-08-31 10:06:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 214.76 +[2026-08-31 10:06:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.22 +[2026-08-31 10:06:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.37 +[2026-08-31 10:06:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 203.89 +[2026-08-31 10:06:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 203.75 +[2026-08-31 10:06:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.18 +[2026-08-31 10:06:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.22 +[2026-08-31 10:07:00 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 195.24 +[2026-08-31 10:07:00 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 195.29 +[2026-08-31 10:07:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 207.77 +[2026-08-31 10:07:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 207.69 +[2026-08-31 10:07:06 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.78 +[2026-08-31 10:07:07 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 268.80 +[2026-08-31 10:07:10 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.98 +[2026-08-31 10:07:10 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.99 +[2026-08-31 10:07:14 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.38 +[2026-08-31 10:07:14 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 204.35 +[2026-08-31 10:07:17 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.70 +[2026-08-31 10:07:17 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 228.63 +[2026-08-31 10:07:20 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.26 +[2026-08-31 10:07:20 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 243.37 +[2026-08-31 10:07:24 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 224.04 +[2026-08-31 10:07:24 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 224.11 +[2026-08-31 10:07:27 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.06 +[2026-08-31 10:07:27 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 179.04 +[2026-08-31 10:07:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.88 +[2026-08-31 10:07:31 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 250.85 +[2026-08-31 10:07:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 164.00 +[2026-08-31 10:07:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.98 +[2026-08-31 10:07:39 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 210.87 +[2026-08-31 10:07:39 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 210.85 +[2026-08-31 10:07:41 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.38 +[2026-08-31 10:07:41 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 344.34 +[2026-08-31 10:07:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.53 +[2026-08-31 10:07:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 332.48 +[2026-08-31 10:07:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.91 +[2026-08-31 10:07:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 319.98 +[2026-08-31 10:07:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 293.98 +[2026-08-31 10:07:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 293.99 +[2026-08-31 10:07:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.43 +[2026-08-31 10:07:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 229.51 +[2026-08-31 10:07:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 360.89 +[2026-08-31 10:07:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 360.66 +[2026-08-31 10:07:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.53 +[2026-08-31 10:07:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 260.62 +[2026-08-31 10:07:59 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.24 +[2026-08-31 10:07:59 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.18 +[2026-08-31 10:08:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 278.89 +[2026-08-31 10:08:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 278.84 +[2026-08-31 10:08:05 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.44 +[2026-08-31 10:08:05 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 237.39 +[2026-08-31 10:08:08 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 244.80 +[2026-08-31 10:08:08 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 244.78 +[2026-08-31 10:08:10 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 336.79 +[2026-08-31 10:08:10 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 337.02 +[2026-08-31 10:08:13 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.88 +[2026-08-31 10:08:13 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.91 +[2026-08-31 10:08:15 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 289.63 +[2026-08-31 10:08:15 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 289.67 +[2026-08-31 10:08:18 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 331.14 +[2026-08-31 10:08:18 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 331.06 +[2026-08-31 10:08:21 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 208.94 +[2026-08-31 10:08:21 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 209.03 +[2026-08-31 10:08:24 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.87 +[2026-08-31 10:08:24 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 269.84 +[2026-08-31 10:08:28 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.42 +[2026-08-31 10:08:28 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 231.36 +[2026-08-31 10:08:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.08 +[2026-08-31 10:08:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 163.08 +[2026-08-31 10:08:34 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 384.65 +[2026-08-31 10:08:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 384.50 +[2026-08-31 10:08:37 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.29 +[2026-08-31 10:08:37 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 275.35 +[2026-08-31 10:08:40 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.39 +[2026-08-31 10:08:40 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 318.36 +[2026-08-31 10:08:44 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.76 +[2026-08-31 10:08:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 189.74 +[2026-08-31 10:08:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.23 +[2026-08-31 10:08:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 185.17 +[2026-08-31 10:08:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.75 +[2026-08-31 10:08:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 197.86 +[2026-08-31 10:08:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.84 +[2026-08-31 10:08:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 198.76 +[2026-08-31 10:08:59 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.20 +[2026-08-31 10:08:59 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 194.29 +[2026-08-31 10:09:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.66 +[2026-08-31 10:09:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 279.66 +[2026-08-31 10:09:05 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.36 +[2026-08-31 10:09:05 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 252.38 +[2026-08-31 10:09:08 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.46 +[2026-08-31 10:09:08 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 273.33 +[2026-08-31 10:09:10 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 303.96 +[2026-08-31 10:09:10 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.02 +[2026-08-31 10:09:13 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.47 +[2026-08-31 10:09:13 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 305.48 +[2026-08-31 10:09:15 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.39 +[2026-08-31 10:09:15 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 271.51 +[2026-08-31 10:09:19 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 246.97 +[2026-08-31 10:09:19 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 246.98 +[2026-08-31 10:09:21 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.73 +[2026-08-31 10:09:21 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 350.72 +[2026-08-31 10:09:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.49 +[2026-08-31 10:09:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 329.47 +[2026-08-31 10:09:26 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.64 +[2026-08-31 10:09:26 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 304.59 +[2026-08-31 10:09:30 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.62 +[2026-08-31 10:09:30 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 174.62 +[2026-08-31 10:09:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 263.51 +[2026-08-31 10:09:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 263.52 +[2026-08-31 10:09:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.90 +[2026-08-31 10:09:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.88 +[2026-08-31 10:09:38 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 302.55 +[2026-08-31 10:09:38 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 302.48 +[2026-08-31 10:09:42 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.15 +[2026-08-31 10:09:42 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 188.17 +[2026-08-31 10:10:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.11 +[2026-08-31 10:10:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19.06 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2467.43 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2194, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3539.13 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19861.88 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 3639, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 25804.08 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 23250.54 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 19068.50 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2937, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 12848.66 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 29535.38 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 2162, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 31229.79 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 24180.13 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 1442, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 34260.95 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 733, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 42001.77 +[2026-08-31 10:10:23 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 37939.24 +[2026-08-31 10:10:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 8, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 34260.87 +[2026-08-31 10:10:26 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.63 +[2026-08-31 10:10:26 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 248.85 +[2026-08-31 10:10:27 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 529.06 +[2026-08-31 10:10:28 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 495.62 +[2026-08-31 10:10:28 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3212.97 +[2026-08-31 10:10:28 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3168.56 +[2026-08-31 10:10:29 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 602.36 +[2026-08-31 10:10:29 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 654.42 +[2026-08-31 10:10:29 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1904.98 +[2026-08-31 10:10:30 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1524.46 +[2026-08-31 10:10:30 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1116.93 +[2026-08-31 10:10:30 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1305.10 +[2026-08-31 10:10:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 718.54 +[2026-08-31 10:10:31 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 721.62 +[2026-08-31 10:10:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 716.12 +[2026-08-31 10:10:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 716.53 +[2026-08-31 10:10:34 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 521.09 +[2026-08-31 10:10:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 519.72 +[2026-08-31 10:10:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 497.56 +[2026-08-31 10:10:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 467.88 +[2026-08-31 10:10:37 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 561.60 +[2026-08-31 10:10:37 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 555.19 +[2026-08-31 10:10:37 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1184.76 +[2026-08-31 10:10:37 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1198.28 +[2026-08-31 10:10:38 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1869.16 +[2026-08-31 10:10:38 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2521.33 +[2026-08-31 10:10:39 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 437.09 +[2026-08-31 10:10:39 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6197.01 +[2026-08-31 10:10:39 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 722, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 411.09 +[2026-08-31 10:10:40 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5101.16 +[2026-08-31 10:10:40 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2835.96 +[2026-08-31 10:10:40 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3322.44 +[2026-08-31 10:10:41 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 728.84 +[2026-08-31 10:10:41 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 814.67 +[2026-08-31 10:10:42 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 441.91 +[2026-08-31 10:10:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 417.20 +[2026-08-31 10:10:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 787.40 +[2026-08-31 10:10:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 787.06 +[2026-08-31 10:10:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 545.85 +[2026-08-31 10:10:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 586.17 +[2026-08-31 10:10:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1330.66 +[2026-08-31 10:10:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 31909.18 +[2026-08-31 10:10:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 734, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1109.76 +[2026-08-31 10:10:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 15792.12 +[2026-08-31 10:10:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1138.43 +[2026-08-31 10:10:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1206.05 +[2026-08-31 10:10:47 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 848.81 +[2026-08-31 10:10:47 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6493.57 +[2026-08-31 10:10:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 708, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 855.64 +[2026-08-31 10:10:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5253.29 +[2026-08-31 10:10:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 862.47 +[2026-08-31 10:10:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1023.40 +[2026-08-31 10:10:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 4722.14 +[2026-08-31 10:10:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3108.08 +[2026-08-31 10:10:49 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 539.64 +[2026-08-31 10:10:50 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 534.56 +[2026-08-31 10:10:50 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3194.56 +[2026-08-31 10:10:50 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3355.40 +[2026-08-31 10:10:51 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 662.27 +[2026-08-31 10:10:51 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 648.50 +[2026-08-31 10:10:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 848.99 +[2026-08-31 10:10:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 846.24 +[2026-08-31 10:10:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1094.15 +[2026-08-31 10:10:53 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 4, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1112.88 +[2026-08-31 10:10:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 643.92 +[2026-08-31 10:10:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 42491.13 +[2026-08-31 10:10:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 702.53 +[2026-08-31 10:10:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 832, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 746, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 24237.42 +[2026-08-31 10:10:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 21161.67 +[2026-08-31 10:10:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 12631.00 +[2026-08-31 10:10:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3729.15 +[2026-08-31 10:10:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 3003.73 +[2026-08-31 10:10:55 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 695.54 +[2026-08-31 10:10:55 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 692.13 +[2026-08-31 10:10:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 947.12 +[2026-08-31 10:10:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1074.42 +[2026-08-31 10:10:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 612.10 +[2026-08-31 10:10:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 569.07 +[2026-08-31 10:10:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 707.48 +[2026-08-31 10:10:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 706.57 +[2026-08-31 10:10:59 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 559.99 +[2026-08-31 10:10:59 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 607.52 +[2026-08-31 10:11:00 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6734.88 +[2026-08-31 10:11:00 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 5339.25 +[2026-08-31 10:11:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 326.16 +[2026-08-31 10:11:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 330.62 +[2026-08-31 10:11:03 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 718.17 +[2026-08-31 10:11:03 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 717.68 +[2026-08-31 10:11:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 628.43 +[2026-08-31 10:11:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 728, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 627.54 +[2026-08-31 10:11:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 34466.58 +[2026-08-31 10:11:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 21396.51 +[2026-08-31 10:11:06 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 508.56 +[2026-08-31 10:11:06 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 713, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 481.22 +[2026-08-31 10:11:06 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6407.01 +[2026-08-31 10:11:06 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6394.03 +[2026-08-31 10:11:07 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 639.78 +[2026-08-31 10:11:07 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 6267.76 +[2026-08-31 10:11:07 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 740, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 637.40 +[2026-08-31 10:11:07 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 28287.12 +[2026-08-31 10:11:08 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1240.37 +[2026-08-31 10:11:08 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1088.93 +[2026-08-31 10:11:08 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1953.55 +[2026-08-31 10:11:08 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1980.62 +[2026-08-31 10:11:09 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 673.01 +[2026-08-31 10:11:09 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 704, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 675.44 +[2026-08-31 10:11:09 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2355.16 +[2026-08-31 10:11:09 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 2217.54 +[2026-08-31 10:11:10 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1813.84 +[2026-08-31 10:11:10 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1855.97 +[2026-08-31 10:11:11 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 978.89 +[2026-08-31 10:11:11 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 1124.97 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/router.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/router.log new file mode 100644 index 0000000..81f0814 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/logs/router.log @@ -0,0 +1,144 @@ +MiniLB is only for debugging purposes, it only supports random policy! +[MiniLB] Overriding policy to random +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:31000 (Press CTRL+C to quit) +INFO: 174.1.60.1:59540 - "GET /health HTTP/1.1" 200 OK +/opt/sglang/lib/python3.12/site-packages/sglang_router/mini_lb.py:396: FastAPIDeprecationWarning: ORJSONResponse is deprecated, FastAPI now serializes data directly to JSON bytes via Pydantic when a return type or response model is set, which is faster and doesn't need a custom response class. Read more in the FastAPI docs: https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model and https://fastapi.tiangolo.com/tutorial/response-model/ + return await lb.generate( +INFO: 174.1.60.1:59548 - "POST /generate HTTP/1.1" 200 OK +/opt/sglang/lib/python3.12/site-packages/sglang_router/mini_lb.py:359: FastAPIDeprecationWarning: ORJSONResponse is deprecated, FastAPI now serializes data directly to JSON bytes via Pydantic when a return type or response model is set, which is faster and doesn't need a custom response class. Read more in the FastAPI docs: https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model and https://fastapi.tiangolo.com/tutorial/response-model/ + return await _get_model_info_impl() +INFO: 174.1.60.1:49872 - "GET /get_model_info HTTP/1.1" 200 OK +INFO: 174.1.60.1:56534 - "GET /health HTTP/1.1" 200 OK +INFO: 174.1.60.1:34528 - "GET /get_model_info HTTP/1.1" 200 OK +INFO: 174.1.60.1:34534 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48724 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48734 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48750 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48764 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48780 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:33686 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:33696 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:50890 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:50902 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:50908 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:60410 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:60424 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:60428 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42190 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42196 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42204 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:40220 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:40236 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:40238 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:59726 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:59742 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:59748 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:44262 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:44274 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:44282 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:44296 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:38590 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:38596 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:38602 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:38618 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42080 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42094 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42102 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:53214 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:53226 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:53228 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:53230 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:46432 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:46434 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58776 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58778 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58790 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58792 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48536 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48548 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58506 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58516 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:58528 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48998 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49000 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49014 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49020 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48172 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48176 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48188 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:48190 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39632 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39646 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39654 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:35784 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:35798 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:35812 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:37916 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45072 - "GET /get_model_info HTTP/1.1" 200 OK +INFO: 174.1.60.1:45078 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45082 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45088 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45102 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45086 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45112 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45098 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:45114 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39922 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39952 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39946 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39934 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39962 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39978 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39986 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42528 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42542 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42568 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42556 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42596 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42606 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42612 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42590 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42618 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42578 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42978 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42968 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43004 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:42992 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43014 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43042 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43022 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43058 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43026 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43062 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43068 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43100 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43084 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49478 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43104 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49490 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49482 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:43120 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49502 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49514 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49524 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49530 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49558 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49542 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49574 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:49580 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39796 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39820 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39812 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39836 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39822 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39850 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39862 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39872 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39898 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39882 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39866 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39906 - "POST /generate HTTP/1.1" 200 OK +INFO: 174.1.60.1:39908 - "POST /generate HTTP/1.1" 200 OK diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/dataset_mount_validation.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/dataset_mount_validation.txt new file mode 100644 index 0000000..9ed689a --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/dataset_mount_validation.txt @@ -0,0 +1 @@ +DATASET_OK 1319 3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/gpu_during_gsm8k.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/gpu_during_gsm8k.txt new file mode 100644 index 0000000..0fbc6f8 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/gpu_during_gsm8k.txt @@ -0,0 +1,80 @@ + +174.1.60.1 2026-08-31T18:08:19+08:00 +0, 53361 MiB, 85651 MiB, 0 % +1, 53361 MiB, 85651 MiB, 0 % +2, 53361 MiB, 85651 MiB, 0 % +3, 53361 MiB, 85651 MiB, 0 % +4, 58677 MiB, 85651 MiB, 0 % +5, 58669 MiB, 85651 MiB, 0 % +6, 58677 MiB, 85651 MiB, 0 % +7, 58669 MiB, 85651 MiB, 0 % + +174.1.60.2 2026-08-31T18:08:20+08:00 +0, 58695 MiB, 85651 MiB, 0 % +1, 58675 MiB, 85651 MiB, 0 % +2, 58695 MiB, 85651 MiB, 0 % +3, 58675 MiB, 85651 MiB, 0 % +4, 63201 MiB, 85651 MiB, 0 % +5, 63201 MiB, 85651 MiB, 0 % +6, 63201 MiB, 85651 MiB, 0 % +7, 63201 MiB, 85651 MiB, 0 % + +174.1.60.3 2026-08-31T18:08:21+08:00 +0, 63367 MiB, 85651 MiB, 94 % +1, 63367 MiB, 85651 MiB, 98 % +2, 63367 MiB, 85651 MiB, 0 % +3, 63367 MiB, 85651 MiB, 14 % +4, 63489 MiB, 85651 MiB, 51 % +5, 63469 MiB, 85651 MiB, 96 % +6, 63469 MiB, 85651 MiB, 0 % +7, 63469 MiB, 85651 MiB, 7 % + +174.1.60.4 2026-08-31T18:08:22+08:00 +0, 63633 MiB, 85651 MiB, 0 % +1, 63633 MiB, 85651 MiB, 0 % +2, 63633 MiB, 85651 MiB, 0 % +3, 63633 MiB, 85651 MiB, 0 % +4, 76579 MiB, 85651 MiB, 0 % +5, 76579 MiB, 85651 MiB, 0 % +6, 76579 MiB, 85651 MiB, 0 % +7, 76579 MiB, 85651 MiB, 0 % + +174.1.60.5 2026-08-31T18:08:23+08:00 +0, 79931 MiB, 85651 MiB, 0 % +1, 79933 MiB, 85651 MiB, 0 % +2, 79917 MiB, 85651 MiB, 0 % +3, 79897 MiB, 85651 MiB, 39 % +4, 79889 MiB, 85651 MiB, 0 % +5, 79933 MiB, 85651 MiB, 0 % +6, 79917 MiB, 85651 MiB, 0 % +7, 80019 MiB, 85651 MiB, 0 % + +174.1.60.6 2026-08-31T18:08:25+08:00 +0, 79971 MiB, 85651 MiB, 98 % +1, 79933 MiB, 85651 MiB, 100 % +2, 79917 MiB, 85651 MiB, 100 % +3, 79937 MiB, 85651 MiB, 100 % +4, 79889 MiB, 85651 MiB, 100 % +5, 79933 MiB, 85651 MiB, 100 % +6, 79917 MiB, 85651 MiB, 92 % +7, 80019 MiB, 85651 MiB, 96 % + +174.1.60.7 2026-08-31T18:08:26+08:00 +0, 79971 MiB, 85651 MiB, 99 % +1, 79933 MiB, 85651 MiB, 100 % +2, 79917 MiB, 85651 MiB, 100 % +3, 79937 MiB, 85651 MiB, 100 % +4, 79889 MiB, 85651 MiB, 100 % +5, 79933 MiB, 85651 MiB, 100 % +6, 79917 MiB, 85651 MiB, 100 % +7, 80019 MiB, 85651 MiB, 99 % + +174.1.60.8 2026-08-31T18:08:27+08:00 +0, 79971 MiB, 85651 MiB, 0 % +1, 79933 MiB, 85651 MiB, 0 % +2, 79917 MiB, 85651 MiB, 0 % +3, 79937 MiB, 85651 MiB, 0 % +4, 79849 MiB, 85651 MiB, 0 % +5, 79933 MiB, 85651 MiB, 0 % +6, 79917 MiB, 85651 MiB, 0 % +7, 79979 MiB, 85651 MiB, 0 % diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/post_run_resources.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/post_run_resources.txt new file mode 100644 index 0000000..31a48d4 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/post_run_resources.txt @@ -0,0 +1,80 @@ + +NODE 174.1.60.1 +0, 3 MiB, 0 % +1, 3 MiB, 0 % +2, 3 MiB, 0 % +3, 3 MiB, 0 % +4, 3 MiB, 0 % +5, 3 MiB, 0 % +6, 3 MiB, 0 % +7, 3 MiB, 0 % + +NODE 174.1.60.2 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % + +NODE 174.1.60.3 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % + +NODE 174.1.60.4 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % + +NODE 174.1.60.5 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % + +NODE 174.1.60.6 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % + +NODE 174.1.60.7 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % + +NODE 174.1.60.8 +0, 0 MiB, 0 % +1, 0 MiB, 0 % +2, 0 MiB, 0 % +3, 0 MiB, 0 % +4, 0 MiB, 0 % +5, 0 MiB, 0 % +6, 0 MiB, 0 % +7, 0 MiB, 0 % diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/preflight.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/preflight.txt new file mode 100644 index 0000000..a4f2334 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/metadata/preflight.txt @@ -0,0 +1,8 @@ +174.1.60.1 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.2 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.3 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.4 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.5 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.6 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.7 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.8 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/result_validation.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/result_validation.json new file mode 100644 index 0000000..708cd4e --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_validation/results/pd-dflash-kvbounds1-20260831-1740/result_validation.json @@ -0,0 +1,246 @@ +{ + "run_id": "pd-dflash-kvbounds1-20260831-1740", + "dataset_sha256": "3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14", + "cases": { + "1": { + "questions": 64, + "successful_requests": 64, + "failed_requests": 0, + "failed_prompt_ids": [], + "num_shots": 5, + "max_new_tokens": 512, + "parallel": 1, + "temperature": 0.0, + "top_p": 1.0, + "prompt_format": "sglang_official_raw_five_shot", + "speculative_algorithm": "DFLASH", + "length_limited_requests": 0, + "accuracy_all_questions": 1.0, + "accuracy_successful_requests": 1.0, + "mean_accept_length_equal_weight_per_question": 6.943340536821033, + "median_accept_length_per_question": 6.2727272727272725, + "min_accept_length_per_question": 4.076923076923077, + "max_accept_length_per_question": 16.333333333333332, + "duration_s": 195.41776519827545, + "output_throughput": 33.379769712245, + "totals": { + "output_tokens": 6523, + "verify_steps": 1026, + "accepted_drafts": 5562, + "proposed_drafts": 15390, + "prompt_tokens": 46392 + }, + "independent_answer_reparse_correct": 64, + "correct_excluding_five_shot_overlap": 59, + "questions_excluding_five_shot_overlap": 59, + "answers_with_final_marker": 64, + "api_accept_length_step_weighted": 6.3576998050682265, + "histogram_accept_length_step_weighted": 6.421052631578948, + "histogram_accept_length_equal_question": 7.03653796026238, + "strict_draft_accept_rate": 0.36140350877192984, + "accept_histogram": [ + 9, + 82, + 139, + 168, + 101, + 116, + 77, + 75, + 64, + 56, + 38, + 24, + 22, + 7, + 8, + 40 + ], + "prompt_tokens_min": 694, + "prompt_tokens_max": 794, + "outputs_max": 225, + "finish_reasons": { + "stop": 64 + }, + "cached_tokens_total": 0, + "retractions_total": 0 + }, + "8": { + "questions": 64, + "successful_requests": 64, + "failed_requests": 0, + "failed_prompt_ids": [], + "num_shots": 5, + "max_new_tokens": 512, + "parallel": 8, + "temperature": 0.0, + "top_p": 1.0, + "prompt_format": "sglang_official_raw_five_shot", + "speculative_algorithm": "DFLASH", + "length_limited_requests": 0, + "accuracy_all_questions": 1.0, + "accuracy_successful_requests": 1.0, + "mean_accept_length_equal_weight_per_question": 7.007018111850178, + "median_accept_length_per_question": 6.35064935064935, + "min_accept_length_per_question": 4.416666666666667, + "max_accept_length_per_question": 16.333333333333332, + "duration_s": 55.545577608048916, + "output_throughput": 117.56111433529787, + "totals": { + "output_tokens": 6530, + "verify_steps": 1012, + "accepted_drafts": 5582, + "proposed_drafts": 15180, + "prompt_tokens": 46392 + }, + "independent_answer_reparse_correct": 64, + "correct_excluding_five_shot_overlap": 59, + "questions_excluding_five_shot_overlap": 59, + "answers_with_final_marker": 64, + "api_accept_length_step_weighted": 6.452569169960475, + "histogram_accept_length_step_weighted": 6.515810276679842, + "histogram_accept_length_equal_question": 7.102034009014687, + "strict_draft_accept_rate": 0.36772068511198946, + "accept_histogram": [ + 10, + 67, + 140, + 169, + 93, + 117, + 76, + 72, + 67, + 60, + 42, + 22, + 20, + 8, + 6, + 43 + ], + "prompt_tokens_min": 694, + "prompt_tokens_max": 794, + "outputs_max": 225, + "finish_reasons": { + "stop": 64 + }, + "cached_tokens_total": 0, + "retractions_total": 0 + } + }, + "log_scan": { + "d_0.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 69, + "rdma_ready_ack": 946, + "graph_decode_batches": 34 + }, + "d_1.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 0, + "rdma_ready_ack": 976, + "graph_decode_batches": 0 + }, + "d_2.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 0, + "rdma_ready_ack": 964, + "graph_decode_batches": 0 + }, + "d_3.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 0, + "rdma_ready_ack": 952, + "graph_decode_batches": 0 + }, + "p_0.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 17, + "rdma_ready_ack": 0, + "graph_decode_batches": 0 + }, + "p_1.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 17, + "rdma_ready_ack": 0, + "graph_decode_batches": 0 + }, + "p_2.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 18, + "rdma_ready_ack": 0, + "graph_decode_batches": 0 + }, + "p_3.log": { + "fatal_signature": 0, + "numa_lookup_warning": 80, + "kda_fused_decode_fallback": 17, + "rdma_ready_ack": 0, + "graph_decode_batches": 0 + }, + "router.log": { + "fatal_signature": 0, + "numa_lookup_warning": 0, + "kda_fused_decode_fallback": 0, + "rdma_ready_ack": 0, + "graph_decode_batches": 0 + } + }, + "metric_definitions": { + "api_accept_length": "completion_tokens / spec_verify_ct, averaged equally over requests", + "histogram_accept_length_weighted": "1 + total accepted draft tokens / total verify steps", + "output_tps": "total completion_tokens / client measured duration_s" + }, + "scope": "fixed 64 GSM8K deployment regression; five-shot examples overlap first 5 questions; not independent held-out accuracy", + "exit_codes": { + "exit_code": 1, + "bench_resume_exit_code": 0, + "cleanup_exit_code": 0 + }, + "evidence_sha256": { + "bench/gsm8k_c1.dataset_path_failure.log": "84d949fb574407d533b7ffbd9350f417f837af9790492db00e93fb5b0129e29d", + "bench/gsm8k_c1.jsonl": "0929e5a8f07aa28955d7a59a9e1685c8d71d81c92ba43d4a61f189e6d4d57b34", + "bench/gsm8k_c1.log": "cc8488e67e2950e02dca977e195977beff21d092fd9f9ca06d240d194b97fa28", + "bench/gsm8k_c1_summary.json": "1831996f6a0c7953118265db03f947720b4072edb963e430fff915258284ccc6", + "bench/gsm8k_c8.jsonl": "11e7be10c8cbcf0dde37d831a434e6d7581fd0574d728b361895396d772de6ee", + "bench/gsm8k_c8.log": "3edfd639473b8009825a0aa19c77b57567af91292a4b106235eb28323c7d9ede", + "bench/gsm8k_c8_summary.json": "2853ecbab21fe9ca0735b54ad470d1038d4681fa954108d68094e18ad68ce109", + "bench/smoke.json": "c32b0f6df0c20565748fb09c579bd80eac9c668800968e3bede315d6a6f51b1b", + "bench_resume_exit_code": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "cleanup_exit_code": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "commands/d_0.cmd.txt": "6f651da18473b3865a83e3bbb29ffc543c4a96ff34488c08fb75f0c84e18fc3e", + "commands/d_1.cmd.txt": "d280afb5a90ba751a1e03277d6b31dcfedcff405397ad176513dad43b31663d7", + "commands/d_2.cmd.txt": "2663f478cd91344d5ff5ae3bd51a49ab9ea71fc70a56b6d07859f2e95ab717d8", + "commands/d_3.cmd.txt": "2bb81aa835ba5c8a18be7eb77811aa4487da0713a7990c913ce025d49e05cd85", + "commands/gsm8k_c1.cmd.txt": "fe4569485e104ae13035064941934d65e9a705d8b260637627b97db8e6e22e30", + "commands/gsm8k_c1.dataset_path_failure.cmd.txt": "0c1aa55b8660afa372cb3290c2377fd7920e374d765bc4d663d3f4626aa8ff8c", + "commands/gsm8k_c8.cmd.txt": "7aa31e99060d18301bb51f9adfa1d62c11f5993caaf1681e9b13a15583de6d8c", + "commands/p_0.cmd.txt": "1147c8d0d1de1925e9f48d38251371c071c92da67e768ac6519b916b803eeb6e", + "commands/p_1.cmd.txt": "261da7b41f0f8c865ee4bdee89710ebf903a4578b6c756a014229e7a7bec8ee4", + "commands/p_2.cmd.txt": "9cf2e4ed7bc337e0d42bd90f7ed8fc03ef3e020a85c4c75bbafc29257ac6ea6a", + "commands/p_3.cmd.txt": "2313aaaadcb1c80652f33a7e0246ace1ab622d4d7e9ecd87b3603247acbdb784", + "commands/router.cmd.txt": "8af366cace8b33785bee4378a9e5a75636858aee249bc16e2dc4f345fab4cfb3", + "exit_code": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "logs/d_0.log": "c846da72e60bcdd4bbf802af7ec7317cb3f5f7c9a4fbff9c7f1584d47e190733", + "logs/d_1.log": "9ec54b0d03d2484c810f72be13efe1e7242a1b077e904ffbfba80f14740d24be", + "logs/d_2.log": "2b71d2b6267914d0caacf82dea1be316030170e7e3f12ecc1484d1d790fea6cb", + "logs/d_3.log": "5dff02037677cdb2f9b9271591e8085e532073224e6b3e9fde8bc893f669a714", + "logs/p_0.log": "30f94f283fa515e7f2929ae32a24134dbb5897ef542a34e7c8c19e170f677f86", + "logs/p_1.log": "bb82b6e4428bc76a1c0e93bd4af6e72b9f22a7647194f4fb807ae08530a7229c", + "logs/p_2.log": "2883931b8784036941f5c05035f4430fd4a722205db82017bcb3fe23dc43adc3", + "logs/p_3.log": "be0548b79556d890fd93d161a97daa67a37cd185413bb107c4f38b6fe5dbe890", + "logs/router.log": "aabfbd56eac2c7d401c707124dcddbb6edc2dae65a1ce9311955d53aacf0e9ce", + "metadata/dataset_mount_validation.txt": "8e33ed2e323b5d1980bae1ad8486ef2a90229f50c5b655a9b96b5cbe489148a3", + "metadata/gpu_during_gsm8k.txt": "5faa28fe4ccd6416f36394853d384f3369fd731c837b836a2e25e5845d9899bd", + "metadata/post_run_resources.txt": "de0d555e14e669190577a076e9288e46e718475b24a3f186832423106becd685", + "metadata/preflight.txt": "eb603bf2006e1cc2f84e5c6d1d6d90a8936b7ea38cdae8c60a1c5170f618574a" + } +}